mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Merge pull request #8011 from Jokser/aws-s3-sdk-integration
AWS SDK S3 integration
This commit is contained in:
alexey-milovidov
GitHub
commit
51ed10ed78
15
.gitmodules
vendored
15
.gitmodules
vendored
@ -107,3 +107,18 @@
|
||||
[submodule "contrib/sparsehash-c11"]
|
||||
path = contrib/sparsehash-c11
|
||||
url = https://github.com/sparsehash/sparsehash-c11.git
|
||||
[submodule "contrib/aws"]
|
||||
path = contrib/aws
|
||||
url = https://github.com/aws/aws-sdk-cpp.git
|
||||
[submodule "aws-c-event-stream"]
|
||||
path = contrib/aws-c-event-stream
|
||||
url = https://github.com/awslabs/aws-c-event-stream.git
|
||||
[submodule "aws-c-common"]
|
||||
path = contrib/aws-c-common
|
||||
url = https://github.com/awslabs/aws-c-common.git
|
||||
[submodule "aws-checksums"]
|
||||
path = contrib/aws-checksums
|
||||
url = https://github.com/awslabs/aws-checksums.git
|
||||
[submodule "contrib/curl"]
|
||||
path = contrib/curl
|
||||
url = https://github.com/curl/curl.git
|
||||
|
||||
@ -325,6 +325,7 @@ include (cmake/find/brotli.cmake)
|
||||
include (cmake/find/protobuf.cmake)
|
||||
include (cmake/find/pdqsort.cmake)
|
||||
include (cmake/find/hdfs3.cmake) # uses protobuf
|
||||
include (cmake/find/s3.cmake)
|
||||
include (cmake/find/consistent-hashing.cmake)
|
||||
include (cmake/find/base64.cmake)
|
||||
include (cmake/find/parquet.cmake)
|
||||
|
||||
26
cmake/find/s3.cmake
Normal file
26
cmake/find/s3.cmake
Normal file
@ -0,0 +1,26 @@
|
||||
if(NOT OS_FREEBSD AND NOT APPLE)
|
||||
option(ENABLE_S3 "Enable S3" ${ENABLE_LIBRARIES})
|
||||
endif()
|
||||
|
||||
if(ENABLE_S3)
|
||||
option(USE_INTERNAL_AWS_S3_LIBRARY "Set to FALSE to use system S3 instead of bundled" ${NOT_UNBUNDLED})
|
||||
|
||||
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3")
|
||||
message (WARNING "submodule contrib/aws is missing. to fix try run: \n git submodule update --init --recursive")
|
||||
set (MISSING_AWS_S3 1)
|
||||
endif ()
|
||||
|
||||
if (USE_INTERNAL_AWS_S3_LIBRARY AND NOT MISSING_AWS_S3)
|
||||
set(AWS_S3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3/include")
|
||||
set(AWS_S3_CORE_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-core/include")
|
||||
set(AWS_S3_LIBRARY aws_s3)
|
||||
set(USE_INTERNAL_AWS_S3_LIBRARY 1)
|
||||
set(USE_AWS_S3 1)
|
||||
else()
|
||||
set(USE_INTERNAL_AWS_S3_LIBRARY 0)
|
||||
set(USE_AWS_S3 0)
|
||||
endif ()
|
||||
|
||||
endif()
|
||||
|
||||
message (STATUS "Using aws_s3=${USE_AWS_S3}: ${AWS_S3_INCLUDE_DIR} : ${AWS_S3_LIBRARY}")
|
||||
15
contrib/CMakeLists.txt
vendored
15
contrib/CMakeLists.txt
vendored
@ -313,6 +313,21 @@ if (USE_INTERNAL_HDFS3_LIBRARY)
|
||||
add_subdirectory(libhdfs3-cmake)
|
||||
endif ()
|
||||
|
||||
if (USE_INTERNAL_AWS_S3_LIBRARY)
|
||||
set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
|
||||
set (save_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
|
||||
set (save_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
|
||||
set (save_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
|
||||
set (save_CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH})
|
||||
add_subdirectory(curl-cmake)
|
||||
set (CMAKE_C_FLAGS ${save_CMAKE_C_FLAGS})
|
||||
set (CMAKE_REQUIRED_LIBRARIES ${save_CMAKE_REQUIRED_LIBRARIES})
|
||||
set (CMAKE_CMAKE_REQUIRED_INCLUDES ${save_CMAKE_REQUIRED_INCLUDES})
|
||||
set (CMAKE_REQUIRED_FLAGS ${save_CMAKE_REQUIRED_FLAGS})
|
||||
set (CMAKE_CMAKE_MODULE_PATH ${save_CMAKE_MODULE_PATH})
|
||||
add_subdirectory(aws-s3-cmake)
|
||||
endif ()
|
||||
|
||||
if (USE_BASE64)
|
||||
add_subdirectory (base64-cmake)
|
||||
endif()
|
||||
|
||||
1
contrib/aws
vendored
Submodule
1
contrib/aws
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 45dd8552d3c492defca79d2720bcc809e35654da
|
||||
1
contrib/aws-c-common
vendored
Submodule
1
contrib/aws-c-common
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 736a82d1697c108b04a277e66438a7f4e19b6857
|
||||
1
contrib/aws-c-event-stream
vendored
Submodule
1
contrib/aws-c-event-stream
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 3bc33662f9ccff4f4cbcf9509cc78c26e022fde0
|
||||
1
contrib/aws-checksums
vendored
Submodule
1
contrib/aws-checksums
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 519d6d9093819b6cf89ffff589a27ef8f83d0f65
|
||||
107
contrib/aws-s3-cmake/CMakeLists.txt
Normal file
107
contrib/aws-s3-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,107 @@
|
||||
SET(AWS_S3_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3)
|
||||
SET(AWS_CORE_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-core)
|
||||
SET(AWS_CHECKSUMS_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws-checksums)
|
||||
SET(AWS_COMMON_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws-c-common)
|
||||
SET(AWS_EVENT_STREAM_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws-c-event-stream)
|
||||
|
||||
OPTION(USE_AWS_MEMORY_MANAGEMENT "Aws memory management" OFF)
|
||||
configure_file("${AWS_CORE_LIBRARY_DIR}/include/aws/core/SDKConfig.h.in"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/include/aws/core/SDKConfig.h" @ONLY)
|
||||
|
||||
configure_file("${AWS_COMMON_LIBRARY_DIR}/include/aws/common/config.h.in"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/include/aws/common/config.h" @ONLY)
|
||||
|
||||
|
||||
file(GLOB AWS_CORE_SOURCES
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/auth/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/client/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/http/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/http/standard/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/http/curl/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/config/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/external/cjson/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/external/tinyxml2/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/internal/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/monitoring/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/net/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/linux-shared/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/platform/linux-shared/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/utils/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/utils/base64/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/utils/event/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/openssl/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/factory/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/utils/json/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/utils/logging/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/utils/memory/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/utils/memory/stl/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/utils/stream/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/utils/threading/*.cpp"
|
||||
"${AWS_CORE_LIBRARY_DIR}/source/utils/xml/*.cpp"
|
||||
)
|
||||
|
||||
file(GLOB AWS_S3_SOURCES
|
||||
"${AWS_S3_LIBRARY_DIR}/source/*.cpp"
|
||||
)
|
||||
|
||||
file(GLOB AWS_S3_MODEL_SOURCES
|
||||
"${AWS_S3_LIBRARY_DIR}/source/model/*.cpp"
|
||||
)
|
||||
|
||||
file(GLOB AWS_EVENT_STREAM_SOURCES
|
||||
"${AWS_EVENT_STREAM_LIBRARY_DIR}/source/*.c"
|
||||
)
|
||||
|
||||
file(GLOB AWS_COMMON_SOURCES
|
||||
"${AWS_COMMON_LIBRARY_DIR}/source/*.c"
|
||||
"${AWS_COMMON_LIBRARY_DIR}/source/posix/*.c"
|
||||
)
|
||||
|
||||
file(GLOB AWS_CHECKSUMS_SOURCES
|
||||
"${AWS_CHECKSUMS_LIBRARY_DIR}/source/*.c"
|
||||
"${AWS_CHECKSUMS_LIBRARY_DIR}/source/intel/*.c"
|
||||
"${AWS_CHECKSUMS_LIBRARY_DIR}/source/arm/*.c"
|
||||
)
|
||||
|
||||
file(GLOB S3_UNIFIED_SRC
|
||||
${AWS_EVENT_STREAM_SOURCES}
|
||||
${AWS_COMMON_SOURCES}
|
||||
${AWS_S3_SOURCES}
|
||||
${AWS_S3_MODEL_SOURCES}
|
||||
${AWS_CORE_SOURCES}
|
||||
)
|
||||
|
||||
set(S3_INCLUDES
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/include/"
|
||||
"${AWS_COMMON_LIBRARY_DIR}/include/"
|
||||
"${AWS_EVENT_STREAM_LIBRARY_DIR}/include/"
|
||||
"${AWS_S3_LIBRARY_DIR}/include/"
|
||||
"${AWS_CORE_LIBRARY_DIR}/include/"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/include/"
|
||||
)
|
||||
|
||||
add_library(aws_s3_checksums ${AWS_CHECKSUMS_SOURCES})
|
||||
target_include_directories(aws_s3_checksums PUBLIC "${AWS_CHECKSUMS_LIBRARY_DIR}/include/")
|
||||
if(CMAKE_BUILD_TYPE STREQUAL "" OR CMAKE_BUILD_TYPE STREQUAL "Debug")
|
||||
target_compile_definitions(aws_s3_checksums PRIVATE "-DDEBUG_BUILD")
|
||||
endif()
|
||||
set_target_properties(aws_s3_checksums PROPERTIES COMPILE_OPTIONS -fPIC)
|
||||
set_target_properties(aws_s3_checksums PROPERTIES LINKER_LANGUAGE C)
|
||||
set_property(TARGET aws_s3_checksums PROPERTY C_STANDARD 99)
|
||||
|
||||
add_library(aws_s3 ${S3_UNIFIED_SRC})
|
||||
|
||||
target_compile_definitions(aws_s3 PUBLIC -DENABLE_CURL_CLIENT)
|
||||
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MAJOR=1")
|
||||
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MINOR=7")
|
||||
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_PATCH=231")
|
||||
target_include_directories(aws_s3 PUBLIC ${S3_INCLUDES} "${CMAKE_BINARY_DIR}/install")
|
||||
|
||||
if (OPENSSL_FOUND)
|
||||
target_compile_definitions(aws_s3 PUBLIC -DENABLE_OPENSSL_ENCRYPTION)
|
||||
target_link_libraries(aws_s3 PRIVATE ${OPENSSL_LIBRARIES})
|
||||
endif()
|
||||
|
||||
target_link_libraries(aws_s3 PRIVATE aws_s3_checksums libcurl)
|
||||
1
contrib/curl
vendored
Submodule
1
contrib/curl
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 3b8bbbbd1609c638a3d3d0acb148a33dedb67be3
|
||||
61
contrib/curl-cmake/CMake/CurlSymbolHiding.cmake
Normal file
61
contrib/curl-cmake/CMake/CurlSymbolHiding.cmake
Normal file
@ -0,0 +1,61 @@
|
||||
include(CheckCSourceCompiles)
|
||||
|
||||
option(CURL_HIDDEN_SYMBOLS "Set to ON to hide libcurl internal symbols (=hide all symbols that aren't officially external)." ON)
|
||||
mark_as_advanced(CURL_HIDDEN_SYMBOLS)
|
||||
|
||||
if(CURL_HIDDEN_SYMBOLS)
|
||||
set(SUPPORTS_SYMBOL_HIDING FALSE)
|
||||
|
||||
if(CMAKE_C_COMPILER_ID MATCHES "Clang")
|
||||
set(SUPPORTS_SYMBOL_HIDING TRUE)
|
||||
set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
|
||||
set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
|
||||
elseif(CMAKE_COMPILER_IS_GNUCC)
|
||||
if(NOT CMAKE_VERSION VERSION_LESS 2.8.10)
|
||||
set(GCC_VERSION ${CMAKE_C_COMPILER_VERSION})
|
||||
else()
|
||||
execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion
|
||||
OUTPUT_VARIABLE GCC_VERSION)
|
||||
endif()
|
||||
if(NOT GCC_VERSION VERSION_LESS 3.4)
|
||||
# note: this is considered buggy prior to 4.0 but the autotools don't care, so let's ignore that fact
|
||||
set(SUPPORTS_SYMBOL_HIDING TRUE)
|
||||
set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
|
||||
set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
|
||||
endif()
|
||||
elseif(CMAKE_C_COMPILER_ID MATCHES "SunPro" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 8.0)
|
||||
set(SUPPORTS_SYMBOL_HIDING TRUE)
|
||||
set(_SYMBOL_EXTERN "__global")
|
||||
set(_CFLAG_SYMBOLS_HIDE "-xldscope=hidden")
|
||||
elseif(CMAKE_C_COMPILER_ID MATCHES "Intel" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0)
|
||||
# note: this should probably just check for version 9.1.045 but I'm not 100% sure
|
||||
# so let's do it the same way autotools do.
|
||||
set(SUPPORTS_SYMBOL_HIDING TRUE)
|
||||
set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
|
||||
set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
|
||||
check_c_source_compiles("#include <stdio.h>
|
||||
int main (void) { printf(\"icc fvisibility bug test\"); return 0; }" _no_bug)
|
||||
if(NOT _no_bug)
|
||||
set(SUPPORTS_SYMBOL_HIDING FALSE)
|
||||
set(_SYMBOL_EXTERN "")
|
||||
set(_CFLAG_SYMBOLS_HIDE "")
|
||||
endif()
|
||||
elseif(MSVC)
|
||||
set(SUPPORTS_SYMBOL_HIDING TRUE)
|
||||
endif()
|
||||
|
||||
set(HIDES_CURL_PRIVATE_SYMBOLS ${SUPPORTS_SYMBOL_HIDING})
|
||||
elseif(MSVC)
|
||||
if(NOT CMAKE_VERSION VERSION_LESS 3.7)
|
||||
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) #present since 3.4.3 but broken
|
||||
set(HIDES_CURL_PRIVATE_SYMBOLS FALSE)
|
||||
else()
|
||||
message(WARNING "Hiding private symbols regardless CURL_HIDDEN_SYMBOLS being disabled.")
|
||||
set(HIDES_CURL_PRIVATE_SYMBOLS TRUE)
|
||||
endif()
|
||||
else()
|
||||
set(HIDES_CURL_PRIVATE_SYMBOLS FALSE)
|
||||
endif()
|
||||
|
||||
set(CURL_CFLAG_SYMBOLS_HIDE ${_CFLAG_SYMBOLS_HIDE})
|
||||
set(CURL_EXTERN_SYMBOL ${_SYMBOL_EXTERN})
|
||||
617
contrib/curl-cmake/CMake/CurlTests.c
Normal file
617
contrib/curl-cmake/CMake/CurlTests.c
Normal file
@ -0,0 +1,617 @@
|
||||
/***************************************************************************
|
||||
* _ _ ____ _
|
||||
* Project ___| | | | _ \| |
|
||||
* / __| | | | |_) | |
|
||||
* | (__| |_| | _ <| |___
|
||||
* \___|\___/|_| \_\_____|
|
||||
*
|
||||
* Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
|
||||
*
|
||||
* This software is licensed as described in the file COPYING, which
|
||||
* you should have received as part of this distribution. The terms
|
||||
* are also available at https://curl.haxx.se/docs/copyright.html.
|
||||
*
|
||||
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
|
||||
* copies of the Software, and permit persons to whom the Software is
|
||||
* furnished to do so, under the terms of the COPYING file.
|
||||
*
|
||||
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
|
||||
* KIND, either express or implied.
|
||||
*
|
||||
***************************************************************************/
|
||||
#ifdef TIME_WITH_SYS_TIME
|
||||
/* Time with sys/time test */
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/time.h>
|
||||
#include <time.h>
|
||||
|
||||
int
|
||||
main ()
|
||||
{
|
||||
if ((struct tm *) 0)
|
||||
return 0;
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_FCNTL_O_NONBLOCK
|
||||
|
||||
/* headers for FCNTL_O_NONBLOCK test */
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
/* */
|
||||
#if defined(sun) || defined(__sun__) || \
|
||||
defined(__SUNPRO_C) || defined(__SUNPRO_CC)
|
||||
# if defined(__SVR4) || defined(__srv4__)
|
||||
# define PLATFORM_SOLARIS
|
||||
# else
|
||||
# define PLATFORM_SUNOS4
|
||||
# endif
|
||||
#endif
|
||||
#if (defined(_AIX) || defined(__xlC__)) && !defined(_AIX41)
|
||||
# define PLATFORM_AIX_V3
|
||||
#endif
|
||||
/* */
|
||||
#if defined(PLATFORM_SUNOS4) || defined(PLATFORM_AIX_V3) || defined(__BEOS__)
|
||||
#error "O_NONBLOCK does not work on this platform"
|
||||
#endif
|
||||
|
||||
int
|
||||
main ()
|
||||
{
|
||||
/* O_NONBLOCK source test */
|
||||
int flags = 0;
|
||||
if(0 != fcntl(0, F_SETFL, flags | O_NONBLOCK))
|
||||
return 1;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* tests for gethostbyaddr_r or gethostbyname_r */
|
||||
#if defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT) || \
|
||||
defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT) || \
|
||||
defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT) || \
|
||||
defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT) || \
|
||||
defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT) || \
|
||||
defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
|
||||
# define _REENTRANT
|
||||
/* no idea whether _REENTRANT is always set, just invent a new flag */
|
||||
# define TEST_GETHOSTBYFOO_REENTRANT
|
||||
#endif
|
||||
#if defined(HAVE_GETHOSTBYADDR_R_5) || \
|
||||
defined(HAVE_GETHOSTBYADDR_R_7) || \
|
||||
defined(HAVE_GETHOSTBYADDR_R_8) || \
|
||||
defined(HAVE_GETHOSTBYNAME_R_3) || \
|
||||
defined(HAVE_GETHOSTBYNAME_R_5) || \
|
||||
defined(HAVE_GETHOSTBYNAME_R_6) || \
|
||||
defined(TEST_GETHOSTBYFOO_REENTRANT)
|
||||
#include <sys/types.h>
|
||||
#include <netdb.h>
|
||||
int main(void)
|
||||
{
|
||||
char *address = "example.com";
|
||||
int length = 0;
|
||||
int type = 0;
|
||||
struct hostent h;
|
||||
int rc = 0;
|
||||
#if defined(HAVE_GETHOSTBYADDR_R_5) || \
|
||||
defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT) || \
|
||||
\
|
||||
defined(HAVE_GETHOSTBYNAME_R_3) || \
|
||||
defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT)
|
||||
struct hostent_data hdata;
|
||||
#elif defined(HAVE_GETHOSTBYADDR_R_7) || \
|
||||
defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT) || \
|
||||
defined(HAVE_GETHOSTBYADDR_R_8) || \
|
||||
defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT) || \
|
||||
\
|
||||
defined(HAVE_GETHOSTBYNAME_R_5) || \
|
||||
defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT) || \
|
||||
defined(HAVE_GETHOSTBYNAME_R_6) || \
|
||||
defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
|
||||
char buffer[8192];
|
||||
int h_errnop;
|
||||
struct hostent *hp;
|
||||
#endif
|
||||
|
||||
#ifndef gethostbyaddr_r
|
||||
(void)gethostbyaddr_r;
|
||||
#endif
|
||||
|
||||
#if defined(HAVE_GETHOSTBYADDR_R_5) || \
|
||||
defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT)
|
||||
rc = gethostbyaddr_r(address, length, type, &h, &hdata);
|
||||
(void)rc;
|
||||
#elif defined(HAVE_GETHOSTBYADDR_R_7) || \
|
||||
defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT)
|
||||
hp = gethostbyaddr_r(address, length, type, &h, buffer, 8192, &h_errnop);
|
||||
(void)hp;
|
||||
#elif defined(HAVE_GETHOSTBYADDR_R_8) || \
|
||||
defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT)
|
||||
rc = gethostbyaddr_r(address, length, type, &h, buffer, 8192, &hp, &h_errnop);
|
||||
(void)rc;
|
||||
#endif
|
||||
|
||||
#if defined(HAVE_GETHOSTBYNAME_R_3) || \
|
||||
defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT)
|
||||
rc = gethostbyname_r(address, &h, &hdata);
|
||||
#elif defined(HAVE_GETHOSTBYNAME_R_5) || \
|
||||
defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT)
|
||||
rc = gethostbyname_r(address, &h, buffer, 8192, &h_errnop);
|
||||
(void)hp; /* not used for test */
|
||||
#elif defined(HAVE_GETHOSTBYNAME_R_6) || \
|
||||
defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
|
||||
rc = gethostbyname_r(address, &h, buffer, 8192, &hp, &h_errnop);
|
||||
#endif
|
||||
|
||||
(void)length;
|
||||
(void)type;
|
||||
(void)rc;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_SOCKLEN_T
|
||||
#ifdef _WIN32
|
||||
#include <ws2tcpip.h>
|
||||
#else
|
||||
#include <sys/types.h>
|
||||
#include <sys/socket.h>
|
||||
#endif
|
||||
int
|
||||
main ()
|
||||
{
|
||||
if ((socklen_t *) 0)
|
||||
return 0;
|
||||
if (sizeof (socklen_t))
|
||||
return 0;
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_IN_ADDR_T
|
||||
#include <sys/types.h>
|
||||
#include <sys/socket.h>
|
||||
#include <arpa/inet.h>
|
||||
|
||||
int
|
||||
main ()
|
||||
{
|
||||
if ((in_addr_t *) 0)
|
||||
return 0;
|
||||
if (sizeof (in_addr_t))
|
||||
return 0;
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_BOOL_T
|
||||
#ifdef HAVE_SYS_TYPES_H
|
||||
#include <sys/types.h>
|
||||
#endif
|
||||
#ifdef HAVE_STDBOOL_H
|
||||
#include <stdbool.h>
|
||||
#endif
|
||||
int
|
||||
main ()
|
||||
{
|
||||
if (sizeof (bool *) )
|
||||
return 0;
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef STDC_HEADERS
|
||||
#include <stdlib.h>
|
||||
#include <stdarg.h>
|
||||
#include <string.h>
|
||||
#include <float.h>
|
||||
int main() { return 0; }
|
||||
#endif
|
||||
#ifdef RETSIGTYPE_TEST
|
||||
#include <sys/types.h>
|
||||
#include <signal.h>
|
||||
#ifdef signal
|
||||
# undef signal
|
||||
#endif
|
||||
#ifdef __cplusplus
|
||||
extern "C" void (*signal (int, void (*)(int)))(int);
|
||||
#else
|
||||
void (*signal ()) ();
|
||||
#endif
|
||||
|
||||
int
|
||||
main ()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_INET_NTOA_R_DECL
|
||||
#include <arpa/inet.h>
|
||||
|
||||
typedef void (*func_type)();
|
||||
|
||||
int main()
|
||||
{
|
||||
#ifndef inet_ntoa_r
|
||||
func_type func;
|
||||
func = (func_type)inet_ntoa_r;
|
||||
(void)func;
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_INET_NTOA_R_DECL_REENTRANT
|
||||
#define _REENTRANT
|
||||
#include <arpa/inet.h>
|
||||
|
||||
typedef void (*func_type)();
|
||||
|
||||
int main()
|
||||
{
|
||||
#ifndef inet_ntoa_r
|
||||
func_type func;
|
||||
func = (func_type)&inet_ntoa_r;
|
||||
(void)func;
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_GETADDRINFO
|
||||
#include <netdb.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/socket.h>
|
||||
|
||||
int main(void) {
|
||||
struct addrinfo hints, *ai;
|
||||
int error;
|
||||
|
||||
memset(&hints, 0, sizeof(hints));
|
||||
hints.ai_family = AF_UNSPEC;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
#ifndef getaddrinfo
|
||||
(void)getaddrinfo;
|
||||
#endif
|
||||
error = getaddrinfo("127.0.0.1", "8080", &hints, &ai);
|
||||
if (error) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_FILE_OFFSET_BITS
|
||||
#ifdef _FILE_OFFSET_BITS
|
||||
#undef _FILE_OFFSET_BITS
|
||||
#endif
|
||||
#define _FILE_OFFSET_BITS 64
|
||||
#include <sys/types.h>
|
||||
/* Check that off_t can represent 2**63 - 1 correctly.
|
||||
We can't simply define LARGE_OFF_T to be 9223372036854775807,
|
||||
since some C++ compilers masquerading as C compilers
|
||||
incorrectly reject 9223372036854775807. */
|
||||
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
|
||||
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
|
||||
&& LARGE_OFF_T % 2147483647 == 1)
|
||||
? 1 : -1];
|
||||
int main () { ; return 0; }
|
||||
#endif
|
||||
#ifdef HAVE_IOCTLSOCKET
|
||||
/* includes start */
|
||||
#ifdef HAVE_WINDOWS_H
|
||||
# ifndef WIN32_LEAN_AND_MEAN
|
||||
# define WIN32_LEAN_AND_MEAN
|
||||
# endif
|
||||
# include <windows.h>
|
||||
# ifdef HAVE_WINSOCK2_H
|
||||
# include <winsock2.h>
|
||||
# else
|
||||
# ifdef HAVE_WINSOCK_H
|
||||
# include <winsock.h>
|
||||
# endif
|
||||
# endif
|
||||
#endif
|
||||
|
||||
int
|
||||
main ()
|
||||
{
|
||||
|
||||
/* ioctlsocket source code */
|
||||
int socket;
|
||||
unsigned long flags = ioctlsocket(socket, FIONBIO, &flags);
|
||||
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif
|
||||
#ifdef HAVE_IOCTLSOCKET_CAMEL
|
||||
/* includes start */
|
||||
#ifdef HAVE_WINDOWS_H
|
||||
# ifndef WIN32_LEAN_AND_MEAN
|
||||
# define WIN32_LEAN_AND_MEAN
|
||||
# endif
|
||||
# include <windows.h>
|
||||
# ifdef HAVE_WINSOCK2_H
|
||||
# include <winsock2.h>
|
||||
# else
|
||||
# ifdef HAVE_WINSOCK_H
|
||||
# include <winsock.h>
|
||||
# endif
|
||||
# endif
|
||||
#endif
|
||||
|
||||
int
|
||||
main ()
|
||||
{
|
||||
|
||||
/* IoctlSocket source code */
|
||||
if(0 != IoctlSocket(0, 0, 0))
|
||||
return 1;
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_IOCTLSOCKET_CAMEL_FIONBIO
|
||||
/* includes start */
|
||||
#ifdef HAVE_WINDOWS_H
|
||||
# ifndef WIN32_LEAN_AND_MEAN
|
||||
# define WIN32_LEAN_AND_MEAN
|
||||
# endif
|
||||
# include <windows.h>
|
||||
# ifdef HAVE_WINSOCK2_H
|
||||
# include <winsock2.h>
|
||||
# else
|
||||
# ifdef HAVE_WINSOCK_H
|
||||
# include <winsock.h>
|
||||
# endif
|
||||
# endif
|
||||
#endif
|
||||
|
||||
int
|
||||
main ()
|
||||
{
|
||||
|
||||
/* IoctlSocket source code */
|
||||
long flags = 0;
|
||||
if(0 != ioctlsocket(0, FIONBIO, &flags))
|
||||
return 1;
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_IOCTLSOCKET_FIONBIO
|
||||
/* includes start */
|
||||
#ifdef HAVE_WINDOWS_H
|
||||
# ifndef WIN32_LEAN_AND_MEAN
|
||||
# define WIN32_LEAN_AND_MEAN
|
||||
# endif
|
||||
# include <windows.h>
|
||||
# ifdef HAVE_WINSOCK2_H
|
||||
# include <winsock2.h>
|
||||
# else
|
||||
# ifdef HAVE_WINSOCK_H
|
||||
# include <winsock.h>
|
||||
# endif
|
||||
# endif
|
||||
#endif
|
||||
|
||||
int
|
||||
main ()
|
||||
{
|
||||
|
||||
int flags = 0;
|
||||
if(0 != ioctlsocket(0, FIONBIO, &flags))
|
||||
return 1;
|
||||
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_IOCTL_FIONBIO
|
||||
/* headers for FIONBIO test */
|
||||
/* includes start */
|
||||
#ifdef HAVE_SYS_TYPES_H
|
||||
# include <sys/types.h>
|
||||
#endif
|
||||
#ifdef HAVE_UNISTD_H
|
||||
# include <unistd.h>
|
||||
#endif
|
||||
#ifdef HAVE_SYS_SOCKET_H
|
||||
# include <sys/socket.h>
|
||||
#endif
|
||||
#ifdef HAVE_SYS_IOCTL_H
|
||||
# include <sys/ioctl.h>
|
||||
#endif
|
||||
#ifdef HAVE_STROPTS_H
|
||||
# include <stropts.h>
|
||||
#endif
|
||||
|
||||
int
|
||||
main ()
|
||||
{
|
||||
|
||||
int flags = 0;
|
||||
if(0 != ioctl(0, FIONBIO, &flags))
|
||||
return 1;
|
||||
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_IOCTL_SIOCGIFADDR
|
||||
/* headers for FIONBIO test */
|
||||
/* includes start */
|
||||
#ifdef HAVE_SYS_TYPES_H
|
||||
# include <sys/types.h>
|
||||
#endif
|
||||
#ifdef HAVE_UNISTD_H
|
||||
# include <unistd.h>
|
||||
#endif
|
||||
#ifdef HAVE_SYS_SOCKET_H
|
||||
# include <sys/socket.h>
|
||||
#endif
|
||||
#ifdef HAVE_SYS_IOCTL_H
|
||||
# include <sys/ioctl.h>
|
||||
#endif
|
||||
#ifdef HAVE_STROPTS_H
|
||||
# include <stropts.h>
|
||||
#endif
|
||||
#include <net/if.h>
|
||||
|
||||
int
|
||||
main ()
|
||||
{
|
||||
struct ifreq ifr;
|
||||
if(0 != ioctl(0, SIOCGIFADDR, &ifr))
|
||||
return 1;
|
||||
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_SETSOCKOPT_SO_NONBLOCK
|
||||
/* includes start */
|
||||
#ifdef HAVE_WINDOWS_H
|
||||
# ifndef WIN32_LEAN_AND_MEAN
|
||||
# define WIN32_LEAN_AND_MEAN
|
||||
# endif
|
||||
# include <windows.h>
|
||||
# ifdef HAVE_WINSOCK2_H
|
||||
# include <winsock2.h>
|
||||
# else
|
||||
# ifdef HAVE_WINSOCK_H
|
||||
# include <winsock.h>
|
||||
# endif
|
||||
# endif
|
||||
#endif
|
||||
/* includes start */
|
||||
#ifdef HAVE_SYS_TYPES_H
|
||||
# include <sys/types.h>
|
||||
#endif
|
||||
#ifdef HAVE_SYS_SOCKET_H
|
||||
# include <sys/socket.h>
|
||||
#endif
|
||||
/* includes end */
|
||||
|
||||
int
|
||||
main ()
|
||||
{
|
||||
if(0 != setsockopt(0, SOL_SOCKET, SO_NONBLOCK, 0, 0))
|
||||
return 1;
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_GLIBC_STRERROR_R
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
|
||||
void check(char c) {}
|
||||
|
||||
int
|
||||
main () {
|
||||
char buffer[1024];
|
||||
/* This will not compile if strerror_r does not return a char* */
|
||||
check(strerror_r(EACCES, buffer, sizeof(buffer))[0]);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_POSIX_STRERROR_R
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
|
||||
/* float, because a pointer can't be implicitly cast to float */
|
||||
void check(float f) {}
|
||||
|
||||
int
|
||||
main () {
|
||||
char buffer[1024];
|
||||
/* This will not compile if strerror_r does not return an int */
|
||||
check(strerror_r(EACCES, buffer, sizeof(buffer)));
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_FSETXATTR_6
|
||||
#include <sys/xattr.h> /* header from libc, not from libattr */
|
||||
int
|
||||
main() {
|
||||
fsetxattr(0, 0, 0, 0, 0, 0);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_FSETXATTR_5
|
||||
#include <sys/xattr.h> /* header from libc, not from libattr */
|
||||
int
|
||||
main() {
|
||||
fsetxattr(0, 0, 0, 0, 0);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_CLOCK_GETTIME_MONOTONIC
|
||||
#include <time.h>
|
||||
int
|
||||
main() {
|
||||
struct timespec ts = {0, 0};
|
||||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_BUILTIN_AVAILABLE
|
||||
int
|
||||
main() {
|
||||
if(__builtin_available(macOS 10.12, *)) {}
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_VARIADIC_MACROS_C99
|
||||
#define c99_vmacro3(first, ...) fun3(first, __VA_ARGS__)
|
||||
#define c99_vmacro2(first, ...) fun2(first, __VA_ARGS__)
|
||||
|
||||
int fun3(int arg1, int arg2, int arg3);
|
||||
int fun2(int arg1, int arg2);
|
||||
|
||||
int fun3(int arg1, int arg2, int arg3) {
|
||||
return arg1 + arg2 + arg3;
|
||||
}
|
||||
int fun2(int arg1, int arg2) {
|
||||
return arg1 + arg2;
|
||||
}
|
||||
|
||||
int
|
||||
main() {
|
||||
int res3 = c99_vmacro3(1, 2, 3);
|
||||
int res2 = c99_vmacro2(1, 2);
|
||||
(void)res3;
|
||||
(void)res2;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#ifdef HAVE_VARIADIC_MACROS_GCC
|
||||
#define gcc_vmacro3(first, args...) fun3(first, args)
|
||||
#define gcc_vmacro2(first, args...) fun2(first, args)
|
||||
|
||||
int fun3(int arg1, int arg2, int arg3);
|
||||
int fun2(int arg1, int arg2);
|
||||
|
||||
int fun3(int arg1, int arg2, int arg3) {
|
||||
return arg1 + arg2 + arg3;
|
||||
}
|
||||
int fun2(int arg1, int arg2) {
|
||||
return arg1 + arg2;
|
||||
}
|
||||
|
||||
int
|
||||
main() {
|
||||
int res3 = gcc_vmacro3(1, 2, 3);
|
||||
int res2 = gcc_vmacro2(1, 2);
|
||||
(void)res3;
|
||||
(void)res2;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
88
contrib/curl-cmake/CMake/Macros.cmake
Normal file
88
contrib/curl-cmake/CMake/Macros.cmake
Normal file
@ -0,0 +1,88 @@
|
||||
#File defines convenience macros for available feature testing
|
||||
|
||||
# This macro checks if the symbol exists in the library and if it
|
||||
# does, it prepends library to the list. It is intended to be called
|
||||
# multiple times with a sequence of possibly dependent libraries in
|
||||
# order of least-to-most-dependent. Some libraries depend on others
|
||||
# to link correctly.
|
||||
macro(check_library_exists_concat LIBRARY SYMBOL VARIABLE)
|
||||
check_library_exists("${LIBRARY};${CURL_LIBS}" ${SYMBOL} "${CMAKE_LIBRARY_PATH}"
|
||||
${VARIABLE})
|
||||
if(${VARIABLE})
|
||||
set(CURL_LIBS ${LIBRARY} ${CURL_LIBS})
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
# Check if header file exists and add it to the list.
|
||||
# This macro is intended to be called multiple times with a sequence of
|
||||
# possibly dependent header files. Some headers depend on others to be
|
||||
# compiled correctly.
|
||||
macro(check_include_file_concat FILE VARIABLE)
|
||||
check_include_files("${CURL_INCLUDES};${FILE}" ${VARIABLE})
|
||||
if(${VARIABLE})
|
||||
set(CURL_INCLUDES ${CURL_INCLUDES} ${FILE})
|
||||
set(CURL_TEST_DEFINES "${CURL_TEST_DEFINES} -D${VARIABLE}")
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
# For other curl specific tests, use this macro.
|
||||
macro(curl_internal_test CURL_TEST)
|
||||
if(NOT DEFINED "${CURL_TEST}")
|
||||
set(MACRO_CHECK_FUNCTION_DEFINITIONS
|
||||
"-D${CURL_TEST} ${CURL_TEST_DEFINES} ${CMAKE_REQUIRED_FLAGS}")
|
||||
if(CMAKE_REQUIRED_LIBRARIES)
|
||||
set(CURL_TEST_ADD_LIBRARIES
|
||||
"-DLINK_LIBRARIES:STRING=${CMAKE_REQUIRED_LIBRARIES}")
|
||||
endif()
|
||||
|
||||
message(STATUS "Performing Curl Test ${CURL_TEST}")
|
||||
try_compile(${CURL_TEST}
|
||||
${CMAKE_BINARY_DIR}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/CMake/CurlTests.c
|
||||
CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_FUNCTION_DEFINITIONS}
|
||||
"${CURL_TEST_ADD_LIBRARIES}"
|
||||
OUTPUT_VARIABLE OUTPUT)
|
||||
if(${CURL_TEST})
|
||||
set(${CURL_TEST} 1 CACHE INTERNAL "Curl test ${FUNCTION}")
|
||||
message(STATUS "Performing Curl Test ${CURL_TEST} - Success")
|
||||
file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log
|
||||
"Performing Curl Test ${CURL_TEST} passed with the following output:\n"
|
||||
"${OUTPUT}\n")
|
||||
else()
|
||||
message(STATUS "Performing Curl Test ${CURL_TEST} - Failed")
|
||||
set(${CURL_TEST} "" CACHE INTERNAL "Curl test ${FUNCTION}")
|
||||
file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log
|
||||
"Performing Curl Test ${CURL_TEST} failed with the following output:\n"
|
||||
"${OUTPUT}\n")
|
||||
endif()
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
macro(curl_nroff_check)
|
||||
find_program(NROFF NAMES gnroff nroff)
|
||||
if(NROFF)
|
||||
# Need a way to write to stdin, this will do
|
||||
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt" "test")
|
||||
# Tests for a valid nroff option to generate a manpage
|
||||
foreach(_MANOPT "-man" "-mandoc")
|
||||
execute_process(COMMAND "${NROFF}" ${_MANOPT}
|
||||
OUTPUT_VARIABLE NROFF_MANOPT_OUTPUT
|
||||
INPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt"
|
||||
ERROR_QUIET)
|
||||
# Save the option if it was valid
|
||||
if(NROFF_MANOPT_OUTPUT)
|
||||
message("Found *nroff option: -- ${_MANOPT}")
|
||||
set(NROFF_MANOPT ${_MANOPT})
|
||||
set(NROFF_USEFUL ON)
|
||||
break()
|
||||
endif()
|
||||
endforeach()
|
||||
# No need for the temporary file
|
||||
file(REMOVE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt")
|
||||
if(NOT NROFF_USEFUL)
|
||||
message(WARNING "Found no *nroff option to get plaintext from man pages")
|
||||
endif()
|
||||
else()
|
||||
message(WARNING "Found no *nroff program")
|
||||
endif()
|
||||
endmacro()
|
||||
264
contrib/curl-cmake/CMake/OtherTests.cmake
Normal file
264
contrib/curl-cmake/CMake/OtherTests.cmake
Normal file
@ -0,0 +1,264 @@
|
||||
include(CheckCSourceCompiles)
|
||||
# The begin of the sources (macros and includes)
|
||||
set(_source_epilogue "#undef inline")
|
||||
|
||||
macro(add_header_include check header)
|
||||
if(${check})
|
||||
set(_source_epilogue "${_source_epilogue}\n#include <${header}>")
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
set(signature_call_conv)
|
||||
if(HAVE_WINDOWS_H)
|
||||
add_header_include(HAVE_WINSOCK2_H "winsock2.h")
|
||||
add_header_include(HAVE_WINDOWS_H "windows.h")
|
||||
add_header_include(HAVE_WINSOCK_H "winsock.h")
|
||||
set(_source_epilogue
|
||||
"${_source_epilogue}\n#ifndef WIN32_LEAN_AND_MEAN\n#define WIN32_LEAN_AND_MEAN\n#endif")
|
||||
set(signature_call_conv "PASCAL")
|
||||
if(HAVE_LIBWS2_32)
|
||||
set(CMAKE_REQUIRED_LIBRARIES ws2_32)
|
||||
endif()
|
||||
else()
|
||||
add_header_include(HAVE_SYS_TYPES_H "sys/types.h")
|
||||
add_header_include(HAVE_SYS_SOCKET_H "sys/socket.h")
|
||||
endif()
|
||||
|
||||
set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
|
||||
|
||||
check_c_source_compiles("${_source_epilogue}
|
||||
int main(void) {
|
||||
recv(0, 0, 0, 0);
|
||||
return 0;
|
||||
}" curl_cv_recv)
|
||||
if(curl_cv_recv)
|
||||
if(NOT DEFINED curl_cv_func_recv_args OR "${curl_cv_func_recv_args}" STREQUAL "unknown")
|
||||
foreach(recv_retv "int" "ssize_t" )
|
||||
foreach(recv_arg1 "SOCKET" "int" )
|
||||
foreach(recv_arg2 "char *" "void *" )
|
||||
foreach(recv_arg3 "int" "size_t" "socklen_t" "unsigned int")
|
||||
foreach(recv_arg4 "int" "unsigned int")
|
||||
if(NOT curl_cv_func_recv_done)
|
||||
unset(curl_cv_func_recv_test CACHE)
|
||||
check_c_source_compiles("
|
||||
${_source_epilogue}
|
||||
extern ${recv_retv} ${signature_call_conv}
|
||||
recv(${recv_arg1}, ${recv_arg2}, ${recv_arg3}, ${recv_arg4});
|
||||
int main(void) {
|
||||
${recv_arg1} s=0;
|
||||
${recv_arg2} buf=0;
|
||||
${recv_arg3} len=0;
|
||||
${recv_arg4} flags=0;
|
||||
${recv_retv} res = recv(s, buf, len, flags);
|
||||
(void) res;
|
||||
return 0;
|
||||
}"
|
||||
curl_cv_func_recv_test)
|
||||
message(STATUS
|
||||
"Tested: ${recv_retv} recv(${recv_arg1}, ${recv_arg2}, ${recv_arg3}, ${recv_arg4})")
|
||||
if(curl_cv_func_recv_test)
|
||||
set(curl_cv_func_recv_args
|
||||
"${recv_arg1},${recv_arg2},${recv_arg3},${recv_arg4},${recv_retv}")
|
||||
set(RECV_TYPE_ARG1 "${recv_arg1}")
|
||||
set(RECV_TYPE_ARG2 "${recv_arg2}")
|
||||
set(RECV_TYPE_ARG3 "${recv_arg3}")
|
||||
set(RECV_TYPE_ARG4 "${recv_arg4}")
|
||||
set(RECV_TYPE_RETV "${recv_retv}")
|
||||
set(HAVE_RECV 1)
|
||||
set(curl_cv_func_recv_done 1)
|
||||
endif()
|
||||
endif()
|
||||
endforeach()
|
||||
endforeach()
|
||||
endforeach()
|
||||
endforeach()
|
||||
endforeach()
|
||||
else()
|
||||
string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG1 "${curl_cv_func_recv_args}")
|
||||
string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG2 "${curl_cv_func_recv_args}")
|
||||
string(REGEX REPLACE "^[^,]*,[^,]*,([^,]*),[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG3 "${curl_cv_func_recv_args}")
|
||||
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,([^,]*),[^,]*$" "\\1" RECV_TYPE_ARG4 "${curl_cv_func_recv_args}")
|
||||
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,([^,]*)$" "\\1" RECV_TYPE_RETV "${curl_cv_func_recv_args}")
|
||||
endif()
|
||||
|
||||
if("${curl_cv_func_recv_args}" STREQUAL "unknown")
|
||||
message(FATAL_ERROR "Cannot find proper types to use for recv args")
|
||||
endif()
|
||||
else()
|
||||
message(FATAL_ERROR "Unable to link function recv")
|
||||
endif()
|
||||
set(curl_cv_func_recv_args "${curl_cv_func_recv_args}" CACHE INTERNAL "Arguments for recv")
|
||||
set(HAVE_RECV 1)
|
||||
|
||||
check_c_source_compiles("${_source_epilogue}
|
||||
int main(void) {
|
||||
send(0, 0, 0, 0);
|
||||
return 0;
|
||||
}" curl_cv_send)
|
||||
if(curl_cv_send)
|
||||
if(NOT DEFINED curl_cv_func_send_args OR "${curl_cv_func_send_args}" STREQUAL "unknown")
|
||||
foreach(send_retv "int" "ssize_t" )
|
||||
foreach(send_arg1 "SOCKET" "int" "ssize_t" )
|
||||
foreach(send_arg2 "const char *" "const void *" "void *" "char *")
|
||||
foreach(send_arg3 "int" "size_t" "socklen_t" "unsigned int")
|
||||
foreach(send_arg4 "int" "unsigned int")
|
||||
if(NOT curl_cv_func_send_done)
|
||||
unset(curl_cv_func_send_test CACHE)
|
||||
check_c_source_compiles("
|
||||
${_source_epilogue}
|
||||
extern ${send_retv} ${signature_call_conv}
|
||||
send(${send_arg1}, ${send_arg2}, ${send_arg3}, ${send_arg4});
|
||||
int main(void) {
|
||||
${send_arg1} s=0;
|
||||
${send_arg2} buf=0;
|
||||
${send_arg3} len=0;
|
||||
${send_arg4} flags=0;
|
||||
${send_retv} res = send(s, buf, len, flags);
|
||||
(void) res;
|
||||
return 0;
|
||||
}"
|
||||
curl_cv_func_send_test)
|
||||
message(STATUS
|
||||
"Tested: ${send_retv} send(${send_arg1}, ${send_arg2}, ${send_arg3}, ${send_arg4})")
|
||||
if(curl_cv_func_send_test)
|
||||
string(REGEX REPLACE "(const) .*" "\\1" send_qual_arg2 "${send_arg2}")
|
||||
string(REGEX REPLACE "const (.*)" "\\1" send_arg2 "${send_arg2}")
|
||||
set(curl_cv_func_send_args
|
||||
"${send_arg1},${send_arg2},${send_arg3},${send_arg4},${send_retv},${send_qual_arg2}")
|
||||
set(SEND_TYPE_ARG1 "${send_arg1}")
|
||||
set(SEND_TYPE_ARG2 "${send_arg2}")
|
||||
set(SEND_TYPE_ARG3 "${send_arg3}")
|
||||
set(SEND_TYPE_ARG4 "${send_arg4}")
|
||||
set(SEND_TYPE_RETV "${send_retv}")
|
||||
set(HAVE_SEND 1)
|
||||
set(curl_cv_func_send_done 1)
|
||||
endif()
|
||||
endif()
|
||||
endforeach()
|
||||
endforeach()
|
||||
endforeach()
|
||||
endforeach()
|
||||
endforeach()
|
||||
else()
|
||||
string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG1 "${curl_cv_func_send_args}")
|
||||
string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG2 "${curl_cv_func_send_args}")
|
||||
string(REGEX REPLACE "^[^,]*,[^,]*,([^,]*),[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG3 "${curl_cv_func_send_args}")
|
||||
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,([^,]*),[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG4 "${curl_cv_func_send_args}")
|
||||
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,([^,]*),[^,]*$" "\\1" SEND_TYPE_RETV "${curl_cv_func_send_args}")
|
||||
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,[^,]*,([^,]*)$" "\\1" SEND_QUAL_ARG2 "${curl_cv_func_send_args}")
|
||||
endif()
|
||||
|
||||
if("${curl_cv_func_send_args}" STREQUAL "unknown")
|
||||
message(FATAL_ERROR "Cannot find proper types to use for send args")
|
||||
endif()
|
||||
set(SEND_QUAL_ARG2 "const")
|
||||
else()
|
||||
message(FATAL_ERROR "Unable to link function send")
|
||||
endif()
|
||||
set(curl_cv_func_send_args "${curl_cv_func_send_args}" CACHE INTERNAL "Arguments for send")
|
||||
set(HAVE_SEND 1)
|
||||
|
||||
check_c_source_compiles("${_source_epilogue}
|
||||
int main(void) {
|
||||
int flag = MSG_NOSIGNAL;
|
||||
(void)flag;
|
||||
return 0;
|
||||
}" HAVE_MSG_NOSIGNAL)
|
||||
|
||||
if(NOT HAVE_WINDOWS_H)
|
||||
add_header_include(HAVE_SYS_TIME_H "sys/time.h")
|
||||
add_header_include(TIME_WITH_SYS_TIME "time.h")
|
||||
add_header_include(HAVE_TIME_H "time.h")
|
||||
endif()
|
||||
check_c_source_compiles("${_source_epilogue}
|
||||
int main(void) {
|
||||
struct timeval ts;
|
||||
ts.tv_sec = 0;
|
||||
ts.tv_usec = 0;
|
||||
(void)ts;
|
||||
return 0;
|
||||
}" HAVE_STRUCT_TIMEVAL)
|
||||
|
||||
set(HAVE_SIG_ATOMIC_T 1)
|
||||
set(CMAKE_REQUIRED_FLAGS)
|
||||
if(HAVE_SIGNAL_H)
|
||||
set(CMAKE_REQUIRED_FLAGS "-DHAVE_SIGNAL_H")
|
||||
set(CMAKE_EXTRA_INCLUDE_FILES "signal.h")
|
||||
endif()
|
||||
check_type_size("sig_atomic_t" SIZEOF_SIG_ATOMIC_T)
|
||||
if(HAVE_SIZEOF_SIG_ATOMIC_T)
|
||||
check_c_source_compiles("
|
||||
#ifdef HAVE_SIGNAL_H
|
||||
# include <signal.h>
|
||||
#endif
|
||||
int main(void) {
|
||||
static volatile sig_atomic_t dummy = 0;
|
||||
(void)dummy;
|
||||
return 0;
|
||||
}" HAVE_SIG_ATOMIC_T_NOT_VOLATILE)
|
||||
if(NOT HAVE_SIG_ATOMIC_T_NOT_VOLATILE)
|
||||
set(HAVE_SIG_ATOMIC_T_VOLATILE 1)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(HAVE_WINDOWS_H)
|
||||
set(CMAKE_EXTRA_INCLUDE_FILES winsock2.h)
|
||||
else()
|
||||
set(CMAKE_EXTRA_INCLUDE_FILES)
|
||||
if(HAVE_SYS_SOCKET_H)
|
||||
set(CMAKE_EXTRA_INCLUDE_FILES sys/socket.h)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
check_type_size("struct sockaddr_storage" SIZEOF_STRUCT_SOCKADDR_STORAGE)
|
||||
if(HAVE_SIZEOF_STRUCT_SOCKADDR_STORAGE)
|
||||
set(HAVE_STRUCT_SOCKADDR_STORAGE 1)
|
||||
endif()
|
||||
|
||||
unset(CMAKE_TRY_COMPILE_TARGET_TYPE)
|
||||
|
||||
if(NOT DEFINED CMAKE_TOOLCHAIN_FILE)
|
||||
# if not cross-compilation...
|
||||
include(CheckCSourceRuns)
|
||||
set(CMAKE_REQUIRED_FLAGS "")
|
||||
if(HAVE_SYS_POLL_H)
|
||||
set(CMAKE_REQUIRED_FLAGS "-DHAVE_SYS_POLL_H")
|
||||
elseif(HAVE_POLL_H)
|
||||
set(CMAKE_REQUIRED_FLAGS "-DHAVE_POLL_H")
|
||||
endif()
|
||||
check_c_source_runs("
|
||||
#include <stdlib.h>
|
||||
#include <sys/time.h>
|
||||
|
||||
#ifdef HAVE_SYS_POLL_H
|
||||
# include <sys/poll.h>
|
||||
#elif HAVE_POLL_H
|
||||
# include <poll.h>
|
||||
#endif
|
||||
|
||||
int main(void)
|
||||
{
|
||||
if(0 != poll(0, 0, 10)) {
|
||||
return 1; /* fail */
|
||||
}
|
||||
else {
|
||||
/* detect the 10.12 poll() breakage */
|
||||
struct timeval before, after;
|
||||
int rc;
|
||||
size_t us;
|
||||
|
||||
gettimeofday(&before, NULL);
|
||||
rc = poll(NULL, 0, 500);
|
||||
gettimeofday(&after, NULL);
|
||||
|
||||
us = (after.tv_sec - before.tv_sec) * 1000000 +
|
||||
(after.tv_usec - before.tv_usec);
|
||||
|
||||
if(us < 400000) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}" HAVE_POLL_FINE)
|
||||
endif()
|
||||
|
||||
28
contrib/curl-cmake/CMake/README.md
Normal file
28
contrib/curl-cmake/CMake/README.md
Normal file
@ -0,0 +1,28 @@
|
||||
#***************************************************************************
|
||||
# _ _ ____ _
|
||||
# Project ___| | | | _ \| |
|
||||
# / __| | | | |_) | |
|
||||
# | (__| |_| | _ <| |___
|
||||
# \___|\___/|_| \_\_____|
|
||||
#
|
||||
# Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
|
||||
#
|
||||
# This software is licensed as described in the file COPYING, which
|
||||
# you should have received as part of this distribution. The terms
|
||||
# are also available at https://curl.haxx.se/docs/copyright.html.
|
||||
#
|
||||
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
|
||||
# copies of the Software, and permit persons to whom the Software is
|
||||
# furnished to do so, under the terms of the COPYING file.
|
||||
#
|
||||
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
|
||||
# KIND, either express or implied.
|
||||
#
|
||||
###########################################################################
|
||||
|
||||
CMake files under this directory were reused from project curl.
|
||||
Here are links to original source files:
|
||||
https://github.com/curl/curl/blob/master/CMake/CurlSymbolHiding.cmake
|
||||
https://github.com/curl/curl/blob/master/CMake/CurlTests,c
|
||||
https://github.com/curl/curl/blob/master/CMake/Macros.cmake
|
||||
https://github.com/curl/curl/blob/master/CMake/OtherTests.cmake
|
||||
1509
contrib/curl-cmake/CMakeLists.patch
Normal file
1509
contrib/curl-cmake/CMakeLists.patch
Normal file
File diff suppressed because it is too large
Load Diff
631
contrib/curl-cmake/CMakeLists.txt
Normal file
631
contrib/curl-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,631 @@
|
||||
#***************************************************************************
|
||||
# _ _ ____ _
|
||||
# Project ___| | | | _ \| |
|
||||
# / __| | | | |_) | |
|
||||
# | (__| |_| | _ <| |___
|
||||
# \___|\___/|_| \_\_____|
|
||||
#
|
||||
# Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
|
||||
#
|
||||
# This software is licensed as described in the file COPYING, which
|
||||
# you should have received as part of this distribution. The terms
|
||||
# are also available at https://curl.haxx.se/docs/copyright.html.
|
||||
#
|
||||
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
|
||||
# copies of the Software, and permit persons to whom the Software is
|
||||
# furnished to do so, under the terms of the COPYING file.
|
||||
#
|
||||
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
|
||||
# KIND, either express or implied.
|
||||
#
|
||||
###########################################################################
|
||||
|
||||
# NOTE:
|
||||
# This file is shrinked and reworked version of original curl CMakeLists.txt
|
||||
# Original file link https://github.com/curl/curl/blob/3b8bbbbd1609c638a3d3d0acb148a33dedb67be3/CMakeLists.txt
|
||||
# If you need to update curl building you can find patch file in this directory
|
||||
# and apply it to fresh original CMakeLists.txt file.
|
||||
cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
|
||||
|
||||
SET(CURL_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl)
|
||||
SET(CURL_LIBRARY_DIR ${CURL_SOURCE_DIR}/lib)
|
||||
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake;${CMAKE_MODULE_PATH}")
|
||||
include(Macros)
|
||||
include(CMakeDependentOption)
|
||||
include(CheckCCompilerFlag)
|
||||
|
||||
file(READ ${CURL_SOURCE_DIR}/include/curl/curlver.h CURL_VERSION_H_CONTENTS)
|
||||
string(REGEX MATCH "#define LIBCURL_VERSION \"[^\"]*"
|
||||
CURL_VERSION ${CURL_VERSION_H_CONTENTS})
|
||||
string(REGEX REPLACE "[^\"]+\"" "" CURL_VERSION ${CURL_VERSION})
|
||||
string(REGEX MATCH "#define LIBCURL_VERSION_NUM 0x[0-9a-fA-F]+"
|
||||
CURL_VERSION_NUM ${CURL_VERSION_H_CONTENTS})
|
||||
string(REGEX REPLACE "[^0]+0x" "" CURL_VERSION_NUM ${CURL_VERSION_NUM})
|
||||
|
||||
message(STATUS "Use curl version=[${CURL_VERSION}]")
|
||||
set(OPERATING_SYSTEM "${CMAKE_SYSTEM_NAME}")
|
||||
set(OS "\"${CMAKE_SYSTEM_NAME}\"")
|
||||
|
||||
option(PICKY_COMPILER "Enable picky compiler options" ON)
|
||||
option(ENABLE_THREADED_RESOLVER "Set to ON to enable threaded DNS lookup" ON)
|
||||
|
||||
if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
|
||||
if(PICKY_COMPILER)
|
||||
foreach(_CCOPT -pedantic -Wall -W -Wpointer-arith -Wwrite-strings -Wunused -Wshadow -Winline -Wnested-externs -Wmissing-declarations -Wmissing-prototypes -Wno-long-long -Wfloat-equal -Wno-multichar -Wsign-compare -Wundef -Wno-format-nonliteral -Wendif-labels -Wstrict-prototypes -Wdeclaration-after-statement -Wstrict-aliasing=3 -Wcast-align -Wtype-limits -Wold-style-declaration -Wmissing-parameter-type -Wempty-body -Wclobbered -Wignored-qualifiers -Wconversion -Wno-sign-conversion -Wvla -Wdouble-promotion -Wno-system-headers -Wno-pedantic-ms-format)
|
||||
# surprisingly, CHECK_C_COMPILER_FLAG needs a new variable to store each new
|
||||
# test result in.
|
||||
check_c_compiler_flag(${_CCOPT} OPT${_CCOPT})
|
||||
if(OPT${_CCOPT})
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_CCOPT}")
|
||||
endif()
|
||||
endforeach()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# For debug libs and exes, add "-d" postfix
|
||||
if(NOT DEFINED CMAKE_DEBUG_POSTFIX)
|
||||
set(CMAKE_DEBUG_POSTFIX "-d")
|
||||
endif()
|
||||
|
||||
# initialize CURL_LIBS
|
||||
set(CURL_LIBS "")
|
||||
|
||||
include(CurlSymbolHiding)
|
||||
|
||||
# Http only
|
||||
set(CURL_DISABLE_FTP ON)
|
||||
set(CURL_DISABLE_LDAP ON)
|
||||
set(CURL_DISABLE_LDAPS ON)
|
||||
set(CURL_DISABLE_TELNET ON)
|
||||
set(CURL_DISABLE_DICT ON)
|
||||
set(CURL_DISABLE_FILE ON)
|
||||
set(CURL_DISABLE_TFTP ON)
|
||||
set(CURL_DISABLE_RTSP ON)
|
||||
set(CURL_DISABLE_POP3 ON)
|
||||
set(CURL_DISABLE_IMAP ON)
|
||||
set(CURL_DISABLE_SMTP ON)
|
||||
set(CURL_DISABLE_GOPHER ON)
|
||||
|
||||
option(CURL_DISABLE_COOKIES "to disable cookies support" OFF)
|
||||
mark_as_advanced(CURL_DISABLE_COOKIES)
|
||||
|
||||
option(CURL_DISABLE_CRYPTO_AUTH "to disable cryptographic authentication" OFF)
|
||||
mark_as_advanced(CURL_DISABLE_CRYPTO_AUTH)
|
||||
|
||||
option(CURL_DISABLE_VERBOSE_STRINGS "to disable verbose strings" OFF)
|
||||
mark_as_advanced(CURL_DISABLE_VERBOSE_STRINGS)
|
||||
|
||||
option(ENABLE_IPV6 "Define if you want to enable IPv6 support" ON)
|
||||
mark_as_advanced(ENABLE_IPV6)
|
||||
|
||||
if(ENABLE_IPV6 AND NOT WIN32)
|
||||
include(CheckStructHasMember)
|
||||
check_struct_has_member("struct sockaddr_in6" sin6_addr "netinet/in.h"
|
||||
HAVE_SOCKADDR_IN6_SIN6_ADDR)
|
||||
check_struct_has_member("struct sockaddr_in6" sin6_scope_id "netinet/in.h"
|
||||
HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID)
|
||||
if(NOT HAVE_SOCKADDR_IN6_SIN6_ADDR)
|
||||
message(WARNING "struct sockaddr_in6 not available, disabling IPv6 support")
|
||||
# Force the feature off as this name is used as guard macro...
|
||||
set(ENABLE_IPV6 OFF
|
||||
CACHE BOOL "Define if you want to enable IPv6 support" FORCE)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
curl_nroff_check()
|
||||
|
||||
# We need ansi c-flags, especially on HP
|
||||
set(CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS}")
|
||||
set(CMAKE_REQUIRED_FLAGS ${CMAKE_ANSI_CFLAGS})
|
||||
|
||||
# Include all the necessary files for macros
|
||||
include(CheckFunctionExists)
|
||||
include(CheckIncludeFile)
|
||||
include(CheckIncludeFiles)
|
||||
include(CheckLibraryExists)
|
||||
include(CheckSymbolExists)
|
||||
include(CheckTypeSize)
|
||||
include(CheckCSourceCompiles)
|
||||
|
||||
if(ENABLE_THREADED_RESOLVER)
|
||||
find_package(Threads REQUIRED)
|
||||
if(WIN32)
|
||||
set(USE_THREADS_WIN32 ON)
|
||||
else()
|
||||
set(USE_THREADS_POSIX ${CMAKE_USE_PTHREADS_INIT})
|
||||
set(HAVE_PTHREAD_H ${CMAKE_USE_PTHREADS_INIT})
|
||||
endif()
|
||||
set(CURL_LIBS ${CURL_LIBS} ${CMAKE_THREAD_LIBS_INIT})
|
||||
endif()
|
||||
|
||||
# Check for all needed libraries
|
||||
check_library_exists_concat("${CMAKE_DL_LIBS}" dlopen HAVE_LIBDL)
|
||||
check_library_exists_concat("socket" connect HAVE_LIBSOCKET)
|
||||
check_library_exists("c" gethostbyname "" NOT_NEED_LIBNSL)
|
||||
|
||||
check_function_exists(gethostname HAVE_GETHOSTNAME)
|
||||
|
||||
# From cmake/find/ssl.cmake
|
||||
if (OPENSSL_FOUND)
|
||||
set(SSL_ENABLED ON)
|
||||
set(USE_OPENSSL ON)
|
||||
|
||||
list(APPEND CURL_LIBS ${OPENSSL_LIBRARIES})
|
||||
check_include_file("openssl/crypto.h" HAVE_OPENSSL_CRYPTO_H)
|
||||
check_include_file("openssl/err.h" HAVE_OPENSSL_ERR_H)
|
||||
check_include_file("openssl/pem.h" HAVE_OPENSSL_PEM_H)
|
||||
check_include_file("openssl/rsa.h" HAVE_OPENSSL_RSA_H)
|
||||
check_include_file("openssl/ssl.h" HAVE_OPENSSL_SSL_H)
|
||||
check_include_file("openssl/x509.h" HAVE_OPENSSL_X509_H)
|
||||
check_include_file("openssl/rand.h" HAVE_OPENSSL_RAND_H)
|
||||
check_symbol_exists(RAND_status "${CURL_INCLUDES}" HAVE_RAND_STATUS)
|
||||
check_symbol_exists(RAND_screen "${CURL_INCLUDES}" HAVE_RAND_SCREEN)
|
||||
check_symbol_exists(RAND_egd "${CURL_INCLUDES}" HAVE_RAND_EGD)
|
||||
endif()
|
||||
|
||||
# Check for idn
|
||||
check_library_exists_concat("idn2" idn2_lookup_ul HAVE_LIBIDN2)
|
||||
|
||||
# Check for symbol dlopen (same as HAVE_LIBDL)
|
||||
check_library_exists("${CURL_LIBS}" dlopen "" HAVE_DLOPEN)
|
||||
|
||||
# From /cmake/find/zlib.cmake
|
||||
if (ZLIB_FOUND)
|
||||
set(HAVE_ZLIB_H ON)
|
||||
set(HAVE_LIBZ ON)
|
||||
set(USE_ZLIB ON)
|
||||
|
||||
list(APPEND CURL_LIBS ${ZLIB_LIBRARIES})
|
||||
endif()
|
||||
|
||||
option(ENABLE_UNIX_SOCKETS "Define if you want Unix domain sockets support" ON)
|
||||
if(ENABLE_UNIX_SOCKETS)
|
||||
include(CheckStructHasMember)
|
||||
check_struct_has_member("struct sockaddr_un" sun_path "sys/un.h" USE_UNIX_SOCKETS)
|
||||
else()
|
||||
unset(USE_UNIX_SOCKETS CACHE)
|
||||
endif()
|
||||
|
||||
# CA handling
|
||||
# Explicitly set to most common case
|
||||
if (OPENSSL_FOUND)
|
||||
set(CURL_CA_BUNDLE "/etc/ssl/certs/ca-certificates.crt")
|
||||
set(CURL_CA_BUNDLE_SET TRUE CACHE BOOL "Path to the CA bundle has been set")
|
||||
set(CURL_CA_PATH "/etc/ssl/certs")
|
||||
set(CURL_CA_PATH_SET TRUE CACHE BOOL "Path to the CA bundle has been set")
|
||||
endif()
|
||||
|
||||
check_include_file_concat("stdio.h" HAVE_STDIO_H)
|
||||
check_include_file_concat("inttypes.h" HAVE_INTTYPES_H)
|
||||
check_include_file_concat("sys/filio.h" HAVE_SYS_FILIO_H)
|
||||
check_include_file_concat("sys/ioctl.h" HAVE_SYS_IOCTL_H)
|
||||
check_include_file_concat("sys/param.h" HAVE_SYS_PARAM_H)
|
||||
check_include_file_concat("sys/poll.h" HAVE_SYS_POLL_H)
|
||||
check_include_file_concat("sys/resource.h" HAVE_SYS_RESOURCE_H)
|
||||
check_include_file_concat("sys/select.h" HAVE_SYS_SELECT_H)
|
||||
check_include_file_concat("sys/socket.h" HAVE_SYS_SOCKET_H)
|
||||
check_include_file_concat("sys/sockio.h" HAVE_SYS_SOCKIO_H)
|
||||
check_include_file_concat("sys/stat.h" HAVE_SYS_STAT_H)
|
||||
check_include_file_concat("sys/time.h" HAVE_SYS_TIME_H)
|
||||
check_include_file_concat("sys/types.h" HAVE_SYS_TYPES_H)
|
||||
check_include_file_concat("sys/uio.h" HAVE_SYS_UIO_H)
|
||||
check_include_file_concat("sys/un.h" HAVE_SYS_UN_H)
|
||||
check_include_file_concat("sys/utime.h" HAVE_SYS_UTIME_H)
|
||||
check_include_file_concat("sys/xattr.h" HAVE_SYS_XATTR_H)
|
||||
check_include_file_concat("alloca.h" HAVE_ALLOCA_H)
|
||||
check_include_file_concat("arpa/inet.h" HAVE_ARPA_INET_H)
|
||||
check_include_file_concat("arpa/tftp.h" HAVE_ARPA_TFTP_H)
|
||||
check_include_file_concat("assert.h" HAVE_ASSERT_H)
|
||||
check_include_file_concat("crypto.h" HAVE_CRYPTO_H)
|
||||
check_include_file_concat("des.h" HAVE_DES_H)
|
||||
check_include_file_concat("err.h" HAVE_ERR_H)
|
||||
check_include_file_concat("errno.h" HAVE_ERRNO_H)
|
||||
check_include_file_concat("fcntl.h" HAVE_FCNTL_H)
|
||||
check_include_file_concat("idn2.h" HAVE_IDN2_H)
|
||||
check_include_file_concat("ifaddrs.h" HAVE_IFADDRS_H)
|
||||
check_include_file_concat("io.h" HAVE_IO_H)
|
||||
check_include_file_concat("krb.h" HAVE_KRB_H)
|
||||
check_include_file_concat("libgen.h" HAVE_LIBGEN_H)
|
||||
check_include_file_concat("locale.h" HAVE_LOCALE_H)
|
||||
check_include_file_concat("net/if.h" HAVE_NET_IF_H)
|
||||
check_include_file_concat("netdb.h" HAVE_NETDB_H)
|
||||
check_include_file_concat("netinet/in.h" HAVE_NETINET_IN_H)
|
||||
check_include_file_concat("netinet/tcp.h" HAVE_NETINET_TCP_H)
|
||||
|
||||
check_include_file_concat("pem.h" HAVE_PEM_H)
|
||||
check_include_file_concat("poll.h" HAVE_POLL_H)
|
||||
check_include_file_concat("pwd.h" HAVE_PWD_H)
|
||||
check_include_file_concat("rsa.h" HAVE_RSA_H)
|
||||
check_include_file_concat("setjmp.h" HAVE_SETJMP_H)
|
||||
check_include_file_concat("sgtty.h" HAVE_SGTTY_H)
|
||||
check_include_file_concat("signal.h" HAVE_SIGNAL_H)
|
||||
check_include_file_concat("ssl.h" HAVE_SSL_H)
|
||||
check_include_file_concat("stdbool.h" HAVE_STDBOOL_H)
|
||||
check_include_file_concat("stdint.h" HAVE_STDINT_H)
|
||||
check_include_file_concat("stdio.h" HAVE_STDIO_H)
|
||||
check_include_file_concat("stdlib.h" HAVE_STDLIB_H)
|
||||
check_include_file_concat("string.h" HAVE_STRING_H)
|
||||
check_include_file_concat("strings.h" HAVE_STRINGS_H)
|
||||
check_include_file_concat("stropts.h" HAVE_STROPTS_H)
|
||||
check_include_file_concat("termio.h" HAVE_TERMIO_H)
|
||||
check_include_file_concat("termios.h" HAVE_TERMIOS_H)
|
||||
check_include_file_concat("time.h" HAVE_TIME_H)
|
||||
check_include_file_concat("unistd.h" HAVE_UNISTD_H)
|
||||
check_include_file_concat("utime.h" HAVE_UTIME_H)
|
||||
check_include_file_concat("x509.h" HAVE_X509_H)
|
||||
|
||||
check_include_file_concat("process.h" HAVE_PROCESS_H)
|
||||
check_include_file_concat("stddef.h" HAVE_STDDEF_H)
|
||||
check_include_file_concat("dlfcn.h" HAVE_DLFCN_H)
|
||||
check_include_file_concat("malloc.h" HAVE_MALLOC_H)
|
||||
check_include_file_concat("memory.h" HAVE_MEMORY_H)
|
||||
check_include_file_concat("netinet/if_ether.h" HAVE_NETINET_IF_ETHER_H)
|
||||
check_include_file_concat("stdint.h" HAVE_STDINT_H)
|
||||
check_include_file_concat("sockio.h" HAVE_SOCKIO_H)
|
||||
check_include_file_concat("sys/utsname.h" HAVE_SYS_UTSNAME_H)
|
||||
|
||||
check_type_size(size_t SIZEOF_SIZE_T)
|
||||
check_type_size(ssize_t SIZEOF_SSIZE_T)
|
||||
check_type_size("long long" SIZEOF_LONG_LONG)
|
||||
check_type_size("long" SIZEOF_LONG)
|
||||
check_type_size("short" SIZEOF_SHORT)
|
||||
check_type_size("int" SIZEOF_INT)
|
||||
check_type_size("__int64" SIZEOF___INT64)
|
||||
check_type_size("long double" SIZEOF_LONG_DOUBLE)
|
||||
check_type_size("time_t" SIZEOF_TIME_T)
|
||||
if(NOT HAVE_SIZEOF_SSIZE_T)
|
||||
if(SIZEOF_LONG EQUAL SIZEOF_SIZE_T)
|
||||
set(ssize_t long)
|
||||
endif()
|
||||
if(NOT ssize_t AND SIZEOF___INT64 EQUAL SIZEOF_SIZE_T)
|
||||
set(ssize_t __int64)
|
||||
endif()
|
||||
endif()
|
||||
# off_t is sized later, after the HAVE_FILE_OFFSET_BITS test
|
||||
|
||||
if(HAVE_SIZEOF_LONG_LONG)
|
||||
set(HAVE_LONGLONG 1)
|
||||
set(HAVE_LL 1)
|
||||
endif()
|
||||
|
||||
find_file(RANDOM_FILE urandom /dev)
|
||||
mark_as_advanced(RANDOM_FILE)
|
||||
|
||||
# Check for some functions that are used
|
||||
if(HAVE_LIBWS2_32)
|
||||
set(CMAKE_REQUIRED_LIBRARIES ws2_32)
|
||||
elseif(HAVE_LIBSOCKET)
|
||||
set(CMAKE_REQUIRED_LIBRARIES socket)
|
||||
endif()
|
||||
|
||||
check_symbol_exists(basename "${CURL_INCLUDES}" HAVE_BASENAME)
|
||||
check_symbol_exists(socket "${CURL_INCLUDES}" HAVE_SOCKET)
|
||||
check_symbol_exists(select "${CURL_INCLUDES}" HAVE_SELECT)
|
||||
check_symbol_exists(poll "${CURL_INCLUDES}" HAVE_POLL)
|
||||
check_symbol_exists(strdup "${CURL_INCLUDES}" HAVE_STRDUP)
|
||||
check_symbol_exists(strstr "${CURL_INCLUDES}" HAVE_STRSTR)
|
||||
check_symbol_exists(strtok_r "${CURL_INCLUDES}" HAVE_STRTOK_R)
|
||||
check_symbol_exists(strftime "${CURL_INCLUDES}" HAVE_STRFTIME)
|
||||
check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME)
|
||||
check_symbol_exists(strcasecmp "${CURL_INCLUDES}" HAVE_STRCASECMP)
|
||||
check_symbol_exists(stricmp "${CURL_INCLUDES}" HAVE_STRICMP)
|
||||
check_symbol_exists(strcmpi "${CURL_INCLUDES}" HAVE_STRCMPI)
|
||||
check_symbol_exists(strncmpi "${CURL_INCLUDES}" HAVE_STRNCMPI)
|
||||
check_symbol_exists(alarm "${CURL_INCLUDES}" HAVE_ALARM)
|
||||
if(NOT HAVE_STRNCMPI)
|
||||
set(HAVE_STRCMPI)
|
||||
endif()
|
||||
check_symbol_exists(gethostbyaddr "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR)
|
||||
check_symbol_exists(gethostbyaddr_r "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR_R)
|
||||
check_symbol_exists(gettimeofday "${CURL_INCLUDES}" HAVE_GETTIMEOFDAY)
|
||||
check_symbol_exists(inet_addr "${CURL_INCLUDES}" HAVE_INET_ADDR)
|
||||
check_symbol_exists(inet_ntoa "${CURL_INCLUDES}" HAVE_INET_NTOA)
|
||||
check_symbol_exists(inet_ntoa_r "${CURL_INCLUDES}" HAVE_INET_NTOA_R)
|
||||
check_symbol_exists(tcsetattr "${CURL_INCLUDES}" HAVE_TCSETATTR)
|
||||
check_symbol_exists(tcgetattr "${CURL_INCLUDES}" HAVE_TCGETATTR)
|
||||
check_symbol_exists(perror "${CURL_INCLUDES}" HAVE_PERROR)
|
||||
check_symbol_exists(closesocket "${CURL_INCLUDES}" HAVE_CLOSESOCKET)
|
||||
check_symbol_exists(setvbuf "${CURL_INCLUDES}" HAVE_SETVBUF)
|
||||
check_symbol_exists(sigsetjmp "${CURL_INCLUDES}" HAVE_SIGSETJMP)
|
||||
check_symbol_exists(getpass_r "${CURL_INCLUDES}" HAVE_GETPASS_R)
|
||||
check_symbol_exists(strlcat "${CURL_INCLUDES}" HAVE_STRLCAT)
|
||||
check_symbol_exists(getpwuid "${CURL_INCLUDES}" HAVE_GETPWUID)
|
||||
check_symbol_exists(getpwuid_r "${CURL_INCLUDES}" HAVE_GETPWUID_R)
|
||||
check_symbol_exists(geteuid "${CURL_INCLUDES}" HAVE_GETEUID)
|
||||
check_symbol_exists(usleep "${CURL_INCLUDES}" HAVE_USLEEP)
|
||||
check_symbol_exists(utime "${CURL_INCLUDES}" HAVE_UTIME)
|
||||
check_symbol_exists(gmtime_r "${CURL_INCLUDES}" HAVE_GMTIME_R)
|
||||
check_symbol_exists(localtime_r "${CURL_INCLUDES}" HAVE_LOCALTIME_R)
|
||||
|
||||
check_symbol_exists(gethostbyname "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME)
|
||||
check_symbol_exists(gethostbyname_r "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME_R)
|
||||
|
||||
check_symbol_exists(signal "${CURL_INCLUDES}" HAVE_SIGNAL_FUNC)
|
||||
check_symbol_exists(SIGALRM "${CURL_INCLUDES}" HAVE_SIGNAL_MACRO)
|
||||
if(HAVE_SIGNAL_FUNC AND HAVE_SIGNAL_MACRO)
|
||||
set(HAVE_SIGNAL 1)
|
||||
endif()
|
||||
check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME)
|
||||
check_symbol_exists(strtoll "${CURL_INCLUDES}" HAVE_STRTOLL)
|
||||
check_symbol_exists(_strtoi64 "${CURL_INCLUDES}" HAVE__STRTOI64)
|
||||
check_symbol_exists(strerror_r "${CURL_INCLUDES}" HAVE_STRERROR_R)
|
||||
check_symbol_exists(siginterrupt "${CURL_INCLUDES}" HAVE_SIGINTERRUPT)
|
||||
check_symbol_exists(perror "${CURL_INCLUDES}" HAVE_PERROR)
|
||||
check_symbol_exists(fork "${CURL_INCLUDES}" HAVE_FORK)
|
||||
check_symbol_exists(getaddrinfo "${CURL_INCLUDES}" HAVE_GETADDRINFO)
|
||||
check_symbol_exists(freeaddrinfo "${CURL_INCLUDES}" HAVE_FREEADDRINFO)
|
||||
check_symbol_exists(freeifaddrs "${CURL_INCLUDES}" HAVE_FREEIFADDRS)
|
||||
check_symbol_exists(pipe "${CURL_INCLUDES}" HAVE_PIPE)
|
||||
check_symbol_exists(ftruncate "${CURL_INCLUDES}" HAVE_FTRUNCATE)
|
||||
check_symbol_exists(getprotobyname "${CURL_INCLUDES}" HAVE_GETPROTOBYNAME)
|
||||
check_symbol_exists(getpeername "${CURL_INCLUDES}" HAVE_GETPEERNAME)
|
||||
check_symbol_exists(getsockname "${CURL_INCLUDES}" HAVE_GETSOCKNAME)
|
||||
check_symbol_exists(if_nametoindex "${CURL_INCLUDES}" HAVE_IF_NAMETOINDEX)
|
||||
check_symbol_exists(getrlimit "${CURL_INCLUDES}" HAVE_GETRLIMIT)
|
||||
check_symbol_exists(setlocale "${CURL_INCLUDES}" HAVE_SETLOCALE)
|
||||
check_symbol_exists(setmode "${CURL_INCLUDES}" HAVE_SETMODE)
|
||||
check_symbol_exists(setrlimit "${CURL_INCLUDES}" HAVE_SETRLIMIT)
|
||||
check_symbol_exists(fcntl "${CURL_INCLUDES}" HAVE_FCNTL)
|
||||
check_symbol_exists(ioctl "${CURL_INCLUDES}" HAVE_IOCTL)
|
||||
check_symbol_exists(setsockopt "${CURL_INCLUDES}" HAVE_SETSOCKOPT)
|
||||
check_function_exists(mach_absolute_time HAVE_MACH_ABSOLUTE_TIME)
|
||||
|
||||
check_symbol_exists(fsetxattr "${CURL_INCLUDES}" HAVE_FSETXATTR)
|
||||
if(HAVE_FSETXATTR)
|
||||
foreach(CURL_TEST HAVE_FSETXATTR_5 HAVE_FSETXATTR_6)
|
||||
curl_internal_test(${CURL_TEST})
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
# sigaction and sigsetjmp are special. Use special mechanism for
|
||||
# detecting those, but only if previous attempt failed.
|
||||
if(HAVE_SIGNAL_H)
|
||||
check_symbol_exists(sigaction "signal.h" HAVE_SIGACTION)
|
||||
endif()
|
||||
|
||||
if(NOT HAVE_SIGSETJMP)
|
||||
if(HAVE_SETJMP_H)
|
||||
check_symbol_exists(sigsetjmp "setjmp.h" HAVE_MACRO_SIGSETJMP)
|
||||
if(HAVE_MACRO_SIGSETJMP)
|
||||
set(HAVE_SIGSETJMP 1)
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# If there is no stricmp(), do not allow LDAP to parse URLs
|
||||
if(NOT HAVE_STRICMP)
|
||||
set(HAVE_LDAP_URL_PARSE 1)
|
||||
endif()
|
||||
|
||||
# Do curl specific tests
|
||||
foreach(CURL_TEST
|
||||
HAVE_FCNTL_O_NONBLOCK
|
||||
HAVE_IOCTLSOCKET
|
||||
HAVE_IOCTLSOCKET_CAMEL
|
||||
HAVE_IOCTLSOCKET_CAMEL_FIONBIO
|
||||
HAVE_IOCTLSOCKET_FIONBIO
|
||||
HAVE_IOCTL_FIONBIO
|
||||
HAVE_IOCTL_SIOCGIFADDR
|
||||
HAVE_SETSOCKOPT_SO_NONBLOCK
|
||||
HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID
|
||||
TIME_WITH_SYS_TIME
|
||||
HAVE_O_NONBLOCK
|
||||
HAVE_GETHOSTBYADDR_R_5
|
||||
HAVE_GETHOSTBYADDR_R_7
|
||||
HAVE_GETHOSTBYADDR_R_8
|
||||
HAVE_GETHOSTBYADDR_R_5_REENTRANT
|
||||
HAVE_GETHOSTBYADDR_R_7_REENTRANT
|
||||
HAVE_GETHOSTBYADDR_R_8_REENTRANT
|
||||
HAVE_GETHOSTBYNAME_R_3
|
||||
HAVE_GETHOSTBYNAME_R_5
|
||||
HAVE_GETHOSTBYNAME_R_6
|
||||
HAVE_GETHOSTBYNAME_R_3_REENTRANT
|
||||
HAVE_GETHOSTBYNAME_R_5_REENTRANT
|
||||
HAVE_GETHOSTBYNAME_R_6_REENTRANT
|
||||
HAVE_IN_ADDR_T
|
||||
HAVE_BOOL_T
|
||||
STDC_HEADERS
|
||||
RETSIGTYPE_TEST
|
||||
HAVE_INET_NTOA_R_DECL
|
||||
HAVE_INET_NTOA_R_DECL_REENTRANT
|
||||
HAVE_GETADDRINFO
|
||||
HAVE_FILE_OFFSET_BITS
|
||||
HAVE_VARIADIC_MACROS_C99
|
||||
HAVE_VARIADIC_MACROS_GCC
|
||||
)
|
||||
curl_internal_test(${CURL_TEST})
|
||||
endforeach()
|
||||
|
||||
if(HAVE_FILE_OFFSET_BITS)
|
||||
set(_FILE_OFFSET_BITS 64)
|
||||
set(CMAKE_REQUIRED_FLAGS "-D_FILE_OFFSET_BITS=64")
|
||||
endif()
|
||||
check_type_size("off_t" SIZEOF_OFF_T)
|
||||
|
||||
# include this header to get the type
|
||||
set(CMAKE_REQUIRED_INCLUDES "${CURL_SOURCE_DIR}/include")
|
||||
set(CMAKE_EXTRA_INCLUDE_FILES "curl/system.h")
|
||||
check_type_size("curl_off_t" SIZEOF_CURL_OFF_T)
|
||||
set(CMAKE_EXTRA_INCLUDE_FILES "")
|
||||
|
||||
foreach(CURL_TEST
|
||||
HAVE_GLIBC_STRERROR_R
|
||||
HAVE_POSIX_STRERROR_R
|
||||
)
|
||||
curl_internal_test(${CURL_TEST})
|
||||
endforeach()
|
||||
|
||||
# Check for reentrant
|
||||
foreach(CURL_TEST
|
||||
HAVE_GETHOSTBYADDR_R_5
|
||||
HAVE_GETHOSTBYADDR_R_7
|
||||
HAVE_GETHOSTBYADDR_R_8
|
||||
HAVE_GETHOSTBYNAME_R_3
|
||||
HAVE_GETHOSTBYNAME_R_5
|
||||
HAVE_GETHOSTBYNAME_R_6
|
||||
HAVE_INET_NTOA_R_DECL_REENTRANT)
|
||||
if(NOT ${CURL_TEST})
|
||||
if(${CURL_TEST}_REENTRANT)
|
||||
set(NEED_REENTRANT 1)
|
||||
endif()
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
if(NEED_REENTRANT)
|
||||
foreach(CURL_TEST
|
||||
HAVE_GETHOSTBYADDR_R_5
|
||||
HAVE_GETHOSTBYADDR_R_7
|
||||
HAVE_GETHOSTBYADDR_R_8
|
||||
HAVE_GETHOSTBYNAME_R_3
|
||||
HAVE_GETHOSTBYNAME_R_5
|
||||
HAVE_GETHOSTBYNAME_R_6)
|
||||
set(${CURL_TEST} 0)
|
||||
if(${CURL_TEST}_REENTRANT)
|
||||
set(${CURL_TEST} 1)
|
||||
endif()
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
if(HAVE_INET_NTOA_R_DECL_REENTRANT)
|
||||
set(HAVE_INET_NTOA_R_DECL 1)
|
||||
set(NEED_REENTRANT 1)
|
||||
endif()
|
||||
|
||||
# Check clock_gettime(CLOCK_MONOTONIC, x) support
|
||||
curl_internal_test(HAVE_CLOCK_GETTIME_MONOTONIC)
|
||||
|
||||
# Check compiler support of __builtin_available()
|
||||
curl_internal_test(HAVE_BUILTIN_AVAILABLE)
|
||||
|
||||
# Some other minor tests
|
||||
|
||||
if(NOT HAVE_IN_ADDR_T)
|
||||
set(in_addr_t "unsigned long")
|
||||
endif()
|
||||
|
||||
# Check for nonblocking
|
||||
set(HAVE_DISABLED_NONBLOCKING 1)
|
||||
if(HAVE_FIONBIO OR
|
||||
HAVE_IOCTLSOCKET OR
|
||||
HAVE_IOCTLSOCKET_CASE OR
|
||||
HAVE_O_NONBLOCK)
|
||||
set(HAVE_DISABLED_NONBLOCKING)
|
||||
endif()
|
||||
|
||||
set(CURL_PULL_SYS_TYPES_H ${HAVE_SYS_TYPES_H})
|
||||
set(CURL_PULL_SYS_SOCKET_H ${HAVE_SYS_SOCKET_H})
|
||||
set(CURL_PULL_SYS_POLL_H ${HAVE_SYS_POLL_H})
|
||||
set(CURL_PULL_STDINT_H ${HAVE_STDINT_H})
|
||||
set(CURL_PULL_INTTYPES_H ${HAVE_INTTYPES_H})
|
||||
|
||||
include(CMake/OtherTests.cmake)
|
||||
|
||||
SET(LIB_VAUTH_CFILES
|
||||
"${CURL_LIBRARY_DIR}/vauth/vauth.c" "${CURL_LIBRARY_DIR}/vauth/cleartext.c" "${CURL_LIBRARY_DIR}/vauth/cram.c"
|
||||
"${CURL_LIBRARY_DIR}/vauth/digest.c" "${CURL_LIBRARY_DIR}/vauth/digest_sspi.c" "${CURL_LIBRARY_DIR}/vauth/krb5_gssapi.c"
|
||||
"${CURL_LIBRARY_DIR}/vauth/krb5_sspi.c" "${CURL_LIBRARY_DIR}/vauth/ntlm.c" "${CURL_LIBRARY_DIR}/vauth/ntlm_sspi.c" "${CURL_LIBRARY_DIR}/vauth/oauth2.c"
|
||||
"${CURL_LIBRARY_DIR}/vauth/spnego_gssapi.c" "${CURL_LIBRARY_DIR}/vauth/spnego_sspi.c")
|
||||
|
||||
SET(LIB_VAUTH_HFILES "${CURL_LIBRARY_DIR}/vauth/vauth.h" "${CURL_LIBRARY_DIR}/vauth/digest.h" "${CURL_LIBRARY_DIR}/vauth/ntlm.h")
|
||||
|
||||
SET(LIB_VTLS_CFILES "${CURL_LIBRARY_DIR}/vtls/openssl.c" "${CURL_LIBRARY_DIR}/vtls/gtls.c" "${CURL_LIBRARY_DIR}/vtls/vtls.c" "${CURL_LIBRARY_DIR}/vtls/nss.c"
|
||||
"${CURL_LIBRARY_DIR}/vtls/polarssl.c" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.c"
|
||||
"${CURL_LIBRARY_DIR}/vtls/wolfssl.c" "${CURL_LIBRARY_DIR}/vtls/schannel.c" "${CURL_LIBRARY_DIR}/vtls/schannel_verify.c"
|
||||
"${CURL_LIBRARY_DIR}/vtls/sectransp.c" "${CURL_LIBRARY_DIR}/vtls/gskit.c" "${CURL_LIBRARY_DIR}/vtls/mbedtls.c" "${CURL_LIBRARY_DIR}/vtls/mesalink.c"
|
||||
"${CURL_LIBRARY_DIR}/vtls/bearssl.c")
|
||||
|
||||
SET(LIB_VTLS_HFILES "${CURL_LIBRARY_DIR}/vtls/openssl.h" "${CURL_LIBRARY_DIR}/vtls/vtls.h" "${CURL_LIBRARY_DIR}/vtls/gtls.h"
|
||||
"${CURL_LIBRARY_DIR}/vtls/nssg.h" "${CURL_LIBRARY_DIR}/vtls/polarssl.h" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.h"
|
||||
"${CURL_LIBRARY_DIR}/vtls/wolfssl.h" "${CURL_LIBRARY_DIR}/vtls/schannel.h" "${CURL_LIBRARY_DIR}/vtls/sectransp.h" "${CURL_LIBRARY_DIR}/vtls/gskit.h"
|
||||
"${CURL_LIBRARY_DIR}/vtls/mbedtls.h" "${CURL_LIBRARY_DIR}/vtls/mesalink.h" "${CURL_LIBRARY_DIR}/vtls/bearssl.h")
|
||||
|
||||
SET(LIB_VQUIC_CFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.c" "${CURL_LIBRARY_DIR}/vquic/quiche.c")
|
||||
|
||||
SET(LIB_VQUIC_HFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.h" "${CURL_LIBRARY_DIR}/vquic/quiche.h")
|
||||
|
||||
SET(LIB_VSSH_CFILES "${CURL_LIBRARY_DIR}/vssh/libssh2.c" "${CURL_LIBRARY_DIR}/vssh/libssh.c")
|
||||
|
||||
SET(LIB_VSSH_HFILES "${CURL_LIBRARY_DIR}/vssh/ssh.h")
|
||||
|
||||
SET(LIB_CFILES "${CURL_LIBRARY_DIR}/file.c"
|
||||
"${CURL_LIBRARY_DIR}/timeval.c" "${CURL_LIBRARY_DIR}/base64.c" "${CURL_LIBRARY_DIR}/hostip.c" "${CURL_LIBRARY_DIR}/progress.c" "${CURL_LIBRARY_DIR}/formdata.c"
|
||||
"${CURL_LIBRARY_DIR}/cookie.c" "${CURL_LIBRARY_DIR}/http.c" "${CURL_LIBRARY_DIR}/sendf.c" "${CURL_LIBRARY_DIR}/url.c" "${CURL_LIBRARY_DIR}/dict.c" "${CURL_LIBRARY_DIR}/if2ip.c" "${CURL_LIBRARY_DIR}/speedcheck.c"
|
||||
"${CURL_LIBRARY_DIR}/ldap.c" "${CURL_LIBRARY_DIR}/version.c" "${CURL_LIBRARY_DIR}/getenv.c" "${CURL_LIBRARY_DIR}/escape.c" "${CURL_LIBRARY_DIR}/mprintf.c" "${CURL_LIBRARY_DIR}/telnet.c" "${CURL_LIBRARY_DIR}/netrc.c"
|
||||
"${CURL_LIBRARY_DIR}/getinfo.c" "${CURL_LIBRARY_DIR}/transfer.c" "${CURL_LIBRARY_DIR}/strcase.c" "${CURL_LIBRARY_DIR}/easy.c" "${CURL_LIBRARY_DIR}/security.c" "${CURL_LIBRARY_DIR}/curl_fnmatch.c"
|
||||
"${CURL_LIBRARY_DIR}/fileinfo.c" "${CURL_LIBRARY_DIR}/wildcard.c" "${CURL_LIBRARY_DIR}/krb5.c" "${CURL_LIBRARY_DIR}/memdebug.c" "${CURL_LIBRARY_DIR}/http_chunks.c"
|
||||
"${CURL_LIBRARY_DIR}/strtok.c" "${CURL_LIBRARY_DIR}/connect.c" "${CURL_LIBRARY_DIR}/llist.c" "${CURL_LIBRARY_DIR}/hash.c" "${CURL_LIBRARY_DIR}/multi.c" "${CURL_LIBRARY_DIR}/content_encoding.c" "${CURL_LIBRARY_DIR}/share.c"
|
||||
"${CURL_LIBRARY_DIR}/http_digest.c" "${CURL_LIBRARY_DIR}/md4.c" "${CURL_LIBRARY_DIR}/md5.c" "${CURL_LIBRARY_DIR}/http_negotiate.c" "${CURL_LIBRARY_DIR}/inet_pton.c" "${CURL_LIBRARY_DIR}/strtoofft.c"
|
||||
"${CURL_LIBRARY_DIR}/strerror.c" "${CURL_LIBRARY_DIR}/amigaos.c" "${CURL_LIBRARY_DIR}/hostasyn.c" "${CURL_LIBRARY_DIR}/hostip4.c" "${CURL_LIBRARY_DIR}/hostip6.c" "${CURL_LIBRARY_DIR}/hostsyn.c"
|
||||
"${CURL_LIBRARY_DIR}/inet_ntop.c" "${CURL_LIBRARY_DIR}/parsedate.c" "${CURL_LIBRARY_DIR}/select.c" "${CURL_LIBRARY_DIR}/splay.c" "${CURL_LIBRARY_DIR}/strdup.c" "${CURL_LIBRARY_DIR}/socks.c"
|
||||
"${CURL_LIBRARY_DIR}/curl_addrinfo.c" "${CURL_LIBRARY_DIR}/socks_gssapi.c" "${CURL_LIBRARY_DIR}/socks_sspi.c"
|
||||
"${CURL_LIBRARY_DIR}/curl_sspi.c" "${CURL_LIBRARY_DIR}/slist.c" "${CURL_LIBRARY_DIR}/nonblock.c" "${CURL_LIBRARY_DIR}/curl_memrchr.c" "${CURL_LIBRARY_DIR}/imap.c" "${CURL_LIBRARY_DIR}/pop3.c" "${CURL_LIBRARY_DIR}/smtp.c"
|
||||
"${CURL_LIBRARY_DIR}/pingpong.c" "${CURL_LIBRARY_DIR}/rtsp.c" "${CURL_LIBRARY_DIR}/curl_threads.c" "${CURL_LIBRARY_DIR}/warnless.c" "${CURL_LIBRARY_DIR}/hmac.c" "${CURL_LIBRARY_DIR}/curl_rtmp.c"
|
||||
"${CURL_LIBRARY_DIR}/openldap.c" "${CURL_LIBRARY_DIR}/curl_gethostname.c" "${CURL_LIBRARY_DIR}/gopher.c" "${CURL_LIBRARY_DIR}/idn_win32.c"
|
||||
"${CURL_LIBRARY_DIR}/http_proxy.c" "${CURL_LIBRARY_DIR}/non-ascii.c" "${CURL_LIBRARY_DIR}/asyn-ares.c" "${CURL_LIBRARY_DIR}/asyn-thread.c" "${CURL_LIBRARY_DIR}/curl_gssapi.c"
|
||||
"${CURL_LIBRARY_DIR}/http_ntlm.c" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.c" "${CURL_LIBRARY_DIR}/curl_ntlm_core.c" "${CURL_LIBRARY_DIR}/curl_sasl.c" "${CURL_LIBRARY_DIR}/rand.c"
|
||||
"${CURL_LIBRARY_DIR}/curl_multibyte.c" "${CURL_LIBRARY_DIR}/hostcheck.c" "${CURL_LIBRARY_DIR}/conncache.c" "${CURL_LIBRARY_DIR}/dotdot.c"
|
||||
"${CURL_LIBRARY_DIR}/x509asn1.c" "${CURL_LIBRARY_DIR}/http2.c" "${CURL_LIBRARY_DIR}/smb.c" "${CURL_LIBRARY_DIR}/curl_endian.c" "${CURL_LIBRARY_DIR}/curl_des.c" "${CURL_LIBRARY_DIR}/system_win32.c"
|
||||
"${CURL_LIBRARY_DIR}/mime.c" "${CURL_LIBRARY_DIR}/sha256.c" "${CURL_LIBRARY_DIR}/setopt.c" "${CURL_LIBRARY_DIR}/curl_path.c" "${CURL_LIBRARY_DIR}/curl_ctype.c" "${CURL_LIBRARY_DIR}/curl_range.c" "${CURL_LIBRARY_DIR}/psl.c"
|
||||
"${CURL_LIBRARY_DIR}/doh.c" "${CURL_LIBRARY_DIR}/urlapi.c" "${CURL_LIBRARY_DIR}/curl_get_line.c" "${CURL_LIBRARY_DIR}/altsvc.c" "${CURL_LIBRARY_DIR}/socketpair.c")
|
||||
|
||||
SET(LIB_HFILES "${CURL_LIBRARY_DIR}/arpa_telnet.h" "${CURL_LIBRARY_DIR}/netrc.h" "${CURL_LIBRARY_DIR}/file.h" "${CURL_LIBRARY_DIR}/timeval.h" "${CURL_LIBRARY_DIR}/hostip.h" "${CURL_LIBRARY_DIR}/progress.h"
|
||||
"${CURL_LIBRARY_DIR}/formdata.h" "${CURL_LIBRARY_DIR}/cookie.h" "${CURL_LIBRARY_DIR}/http.h" "${CURL_LIBRARY_DIR}/sendf.h" "${CURL_LIBRARY_DIR}/url.h" "${CURL_LIBRARY_DIR}/dict.h" "${CURL_LIBRARY_DIR}/if2ip.h"
|
||||
"${CURL_LIBRARY_DIR}/speedcheck.h" "${CURL_LIBRARY_DIR}/urldata.h" "${CURL_LIBRARY_DIR}/curl_ldap.h" "${CURL_LIBRARY_DIR}/escape.h" "${CURL_LIBRARY_DIR}/telnet.h" "${CURL_LIBRARY_DIR}/getinfo.h"
|
||||
"${CURL_LIBRARY_DIR}/strcase.h" "${CURL_LIBRARY_DIR}/curl_sec.h" "${CURL_LIBRARY_DIR}/memdebug.h" "${CURL_LIBRARY_DIR}/http_chunks.h" "${CURL_LIBRARY_DIR}/curl_fnmatch.h"
|
||||
"${CURL_LIBRARY_DIR}/wildcard.h" "${CURL_LIBRARY_DIR}/fileinfo.h" "${CURL_LIBRARY_DIR}/strtok.h" "${CURL_LIBRARY_DIR}/connect.h" "${CURL_LIBRARY_DIR}/llist.h"
|
||||
"${CURL_LIBRARY_DIR}/hash.h" "${CURL_LIBRARY_DIR}/content_encoding.h" "${CURL_LIBRARY_DIR}/share.h" "${CURL_LIBRARY_DIR}/curl_md4.h" "${CURL_LIBRARY_DIR}/curl_md5.h" "${CURL_LIBRARY_DIR}/http_digest.h"
|
||||
"${CURL_LIBRARY_DIR}/http_negotiate.h" "${CURL_LIBRARY_DIR}/inet_pton.h" "${CURL_LIBRARY_DIR}/amigaos.h" "${CURL_LIBRARY_DIR}/strtoofft.h" "${CURL_LIBRARY_DIR}/strerror.h"
|
||||
"${CURL_LIBRARY_DIR}/inet_ntop.h" "${CURL_LIBRARY_DIR}/curlx.h" "${CURL_LIBRARY_DIR}/curl_memory.h" "${CURL_LIBRARY_DIR}/curl_setup.h" "${CURL_LIBRARY_DIR}/transfer.h" "${CURL_LIBRARY_DIR}/select.h"
|
||||
"${CURL_LIBRARY_DIR}/easyif.h" "${CURL_LIBRARY_DIR}/multiif.h" "${CURL_LIBRARY_DIR}/parsedate.h" "${CURL_LIBRARY_DIR}/sockaddr.h" "${CURL_LIBRARY_DIR}/splay.h" "${CURL_LIBRARY_DIR}/strdup.h"
|
||||
"${CURL_LIBRARY_DIR}/socks.h" "${CURL_LIBRARY_DIR}/curl_base64.h" "${CURL_LIBRARY_DIR}/curl_addrinfo.h" "${CURL_LIBRARY_DIR}/curl_sspi.h"
|
||||
"${CURL_LIBRARY_DIR}/slist.h" "${CURL_LIBRARY_DIR}/nonblock.h" "${CURL_LIBRARY_DIR}/curl_memrchr.h" "${CURL_LIBRARY_DIR}/imap.h" "${CURL_LIBRARY_DIR}/pop3.h" "${CURL_LIBRARY_DIR}/smtp.h" "${CURL_LIBRARY_DIR}/pingpong.h"
|
||||
"${CURL_LIBRARY_DIR}/rtsp.h" "${CURL_LIBRARY_DIR}/curl_threads.h" "${CURL_LIBRARY_DIR}/warnless.h" "${CURL_LIBRARY_DIR}/curl_hmac.h" "${CURL_LIBRARY_DIR}/curl_rtmp.h"
|
||||
"${CURL_LIBRARY_DIR}/curl_gethostname.h" "${CURL_LIBRARY_DIR}/gopher.h" "${CURL_LIBRARY_DIR}/http_proxy.h" "${CURL_LIBRARY_DIR}/non-ascii.h" "${CURL_LIBRARY_DIR}/asyn.h"
|
||||
"${CURL_LIBRARY_DIR}/http_ntlm.h" "${CURL_LIBRARY_DIR}/curl_gssapi.h" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.h" "${CURL_LIBRARY_DIR}/curl_ntlm_core.h"
|
||||
"${CURL_LIBRARY_DIR}/curl_sasl.h" "${CURL_LIBRARY_DIR}/curl_multibyte.h" "${CURL_LIBRARY_DIR}/hostcheck.h" "${CURL_LIBRARY_DIR}/conncache.h"
|
||||
"${CURL_LIBRARY_DIR}/multihandle.h" "${CURL_LIBRARY_DIR}/setup-vms.h" "${CURL_LIBRARY_DIR}/dotdot.h"
|
||||
"${CURL_LIBRARY_DIR}/x509asn1.h" "${CURL_LIBRARY_DIR}/http2.h" "${CURL_LIBRARY_DIR}/sigpipe.h" "${CURL_LIBRARY_DIR}/smb.h" "${CURL_LIBRARY_DIR}/curl_endian.h" "${CURL_LIBRARY_DIR}/curl_des.h"
|
||||
"${CURL_LIBRARY_DIR}/curl_printf.h" "${CURL_LIBRARY_DIR}/system_win32.h" "${CURL_LIBRARY_DIR}/rand.h" "${CURL_LIBRARY_DIR}/mime.h" "${CURL_LIBRARY_DIR}/curl_sha256.h" "${CURL_LIBRARY_DIR}/setopt.h"
|
||||
"${CURL_LIBRARY_DIR}/curl_path.h" "${CURL_LIBRARY_DIR}/curl_ctype.h" "${CURL_LIBRARY_DIR}/curl_range.h" "${CURL_LIBRARY_DIR}/psl.h" "${CURL_LIBRARY_DIR}/doh.h" "${CURL_LIBRARY_DIR}/urlapi-int.h"
|
||||
"${CURL_LIBRARY_DIR}/curl_get_line.h" "${CURL_LIBRARY_DIR}/altsvc.h" "${CURL_LIBRARY_DIR}/quic.h" "${CURL_LIBRARY_DIR}/socketpair.h")
|
||||
|
||||
SET(LIB_RCFILES "${CURL_LIBRARY_DIR}/libcurl.rc")
|
||||
|
||||
SET(CSOURCES ${LIB_CFILES} ${LIB_VAUTH_CFILES} ${LIB_VTLS_CFILES}
|
||||
${LIB_VQUIC_CFILES} ${LIB_VSSH_CFILES})
|
||||
SET(HHEADERS ${LIB_HFILES} ${LIB_VAUTH_HFILES} ${LIB_VTLS_HFILES}
|
||||
${LIB_VQUIC_HFILES} ${LIB_VSSH_HFILES})
|
||||
|
||||
configure_file(${CURL_SOURCE_DIR}/lib/curl_config.h.cmake
|
||||
${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h)
|
||||
|
||||
list(APPEND HHEADERS
|
||||
${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h
|
||||
)
|
||||
|
||||
add_library(libcurl ${HHEADERS} ${CSOURCES})
|
||||
|
||||
if(NOT BUILD_SHARED_LIBS)
|
||||
set_target_properties(libcurl PROPERTIES INTERFACE_COMPILE_DEFINITIONS CURL_STATICLIB)
|
||||
endif()
|
||||
|
||||
if(HIDES_CURL_PRIVATE_SYMBOLS)
|
||||
set_property(TARGET libcurl APPEND PROPERTY COMPILE_DEFINITIONS "CURL_HIDDEN_SYMBOLS")
|
||||
set_property(TARGET libcurl APPEND PROPERTY COMPILE_FLAGS ${CURL_CFLAG_SYMBOLS_HIDE})
|
||||
endif()
|
||||
|
||||
if(OPENSSL_FOUND)
|
||||
target_include_directories(libcurl PUBLIC ${OPENSSL_INCLUDE_DIR})
|
||||
message("-- Including openssl ${OPENSSL_INCLUDE_DIR} to curl")
|
||||
endif()
|
||||
|
||||
if(ZLIB_FOUND)
|
||||
target_include_directories(libcurl PUBLIC ${ZLIB_INCLUDE_DIRS}})
|
||||
message("-- Including zlib ${ZLIB_INCLUDE_DIRS} to curl")
|
||||
endif()
|
||||
|
||||
target_compile_definitions(libcurl PUBLIC -DHAVE_CONFIG_H)
|
||||
target_compile_definitions(libcurl PUBLIC -DBUILDING_LIBCURL)
|
||||
target_include_directories(libcurl PUBLIC "${CURL_SOURCE_DIR}/include" "${CURL_LIBRARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/curl")
|
||||
|
||||
target_link_libraries(libcurl ${CURL_LIBS})
|
||||
@ -432,6 +432,12 @@ if (USE_HDFS)
|
||||
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${HDFS3_INCLUDE_DIR})
|
||||
endif()
|
||||
|
||||
if (USE_AWS_S3)
|
||||
target_link_libraries (clickhouse_common_io PUBLIC ${AWS_S3_LIBRARY})
|
||||
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AWS_S3_CORE_INCLUDE_DIR})
|
||||
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AWS_S3_INCLUDE_DIR})
|
||||
endif()
|
||||
|
||||
if (USE_BROTLI)
|
||||
target_link_libraries (clickhouse_common_io PRIVATE ${BROTLI_LIBRARY})
|
||||
target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BROTLI_INCLUDE_DIR})
|
||||
|
||||
@ -473,6 +473,7 @@ namespace ErrorCodes
|
||||
extern const int QUOTA_REQUIRES_CLIENT_KEY = 496;
|
||||
extern const int NOT_ENOUGH_PRIVILEGES = 497;
|
||||
extern const int LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED = 498;
|
||||
extern const int S3_ERROR = 499;
|
||||
|
||||
extern const int KEEPER_EXCEPTION = 999;
|
||||
extern const int POCO_EXCEPTION = 1000;
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#cmakedefine01 USE_SSL
|
||||
#cmakedefine01 USE_POCO_NETSSL
|
||||
#cmakedefine01 USE_HDFS
|
||||
#cmakedefine01 USE_AWS_S3
|
||||
#cmakedefine01 USE_CPUID
|
||||
#cmakedefine01 USE_CPUINFO
|
||||
#cmakedefine01 USE_BROTLI
|
||||
|
||||
@ -1,68 +1,43 @@
|
||||
#include <IO/ReadBufferFromS3.h>
|
||||
#include <Common/config.h>
|
||||
|
||||
#if USE_AWS_S3
|
||||
|
||||
#include <IO/ReadBufferFromS3.h>
|
||||
#include <IO/ReadBufferFromIStream.h>
|
||||
#include <IO/S3Common.h>
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
#include <aws/s3/model/GetObjectRequest.h>
|
||||
#include <aws/s3/S3Client.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
const int DEFAULT_S3_MAX_FOLLOW_GET_REDIRECT = 2;
|
||||
|
||||
ReadBufferFromS3::ReadBufferFromS3(const Poco::URI & uri_,
|
||||
const String & access_key_id_,
|
||||
const String & secret_access_key_,
|
||||
const ConnectionTimeouts & timeouts,
|
||||
const RemoteHostFilter & remote_host_filter_)
|
||||
: ReadBuffer(nullptr, 0)
|
||||
, uri {uri_}
|
||||
, session {makeHTTPSession(uri_, timeouts)}
|
||||
, remote_host_filter {remote_host_filter_}
|
||||
namespace ErrorCodes
|
||||
{
|
||||
Poco::Net::HTTPResponse response;
|
||||
std::unique_ptr<Poco::Net::HTTPRequest> request;
|
||||
|
||||
for (int i = 0; i < DEFAULT_S3_MAX_FOLLOW_GET_REDIRECT; ++i)
|
||||
{
|
||||
// With empty path poco will send "POST HTTP/1.1" its bug.
|
||||
if (uri.getPath().empty())
|
||||
uri.setPath("/");
|
||||
|
||||
request = std::make_unique<Poco::Net::HTTPRequest>(
|
||||
Poco::Net::HTTPRequest::HTTP_GET,
|
||||
uri.getPathAndQuery(),
|
||||
Poco::Net::HTTPRequest::HTTP_1_1);
|
||||
request->setHost(uri.getHost()); // use original, not resolved host name in header
|
||||
|
||||
S3Helper::authenticateRequest(*request, access_key_id_, secret_access_key_);
|
||||
|
||||
LOG_TRACE((&Logger::get("ReadBufferFromS3")), "Sending request to " << uri.toString());
|
||||
|
||||
session->sendRequest(*request);
|
||||
|
||||
istr = &session->receiveResponse(response);
|
||||
|
||||
// Handle 307 Temporary Redirect in order to allow request redirection
|
||||
// See https://docs.aws.amazon.com/AmazonS3/latest/dev/Redirects.html
|
||||
if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_TEMPORARY_REDIRECT)
|
||||
break;
|
||||
|
||||
auto location_iterator = response.find("Location");
|
||||
if (location_iterator == response.end())
|
||||
break;
|
||||
|
||||
uri = location_iterator->second;
|
||||
remote_host_filter.checkURL(uri);
|
||||
session = makeHTTPSession(uri, timeouts);
|
||||
}
|
||||
|
||||
assertResponseIsOk(*request, response, *istr);
|
||||
impl = std::make_unique<ReadBufferFromIStream>(*istr, DBMS_DEFAULT_BUFFER_SIZE);
|
||||
extern const int S3_ERROR;
|
||||
}
|
||||
|
||||
|
||||
ReadBufferFromS3::ReadBufferFromS3(const std::shared_ptr<Aws::S3::S3Client> & client_ptr,
|
||||
const String & bucket,
|
||||
const String & key,
|
||||
size_t buffer_size_): ReadBuffer(nullptr, 0)
|
||||
{
|
||||
Aws::S3::Model::GetObjectRequest req;
|
||||
req.SetBucket(bucket);
|
||||
req.SetKey(key);
|
||||
|
||||
Aws::S3::Model::GetObjectOutcome outcome = client_ptr->GetObject(req);
|
||||
|
||||
if (outcome.IsSuccess())
|
||||
{
|
||||
read_result = outcome.GetResultWithOwnership();
|
||||
impl = std::make_unique<ReadBufferFromIStream>(read_result.GetBody(), buffer_size_);
|
||||
}
|
||||
else
|
||||
throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
|
||||
}
|
||||
|
||||
bool ReadBufferFromS3::nextImpl()
|
||||
{
|
||||
if (!impl->next())
|
||||
@ -73,3 +48,5 @@ bool ReadBufferFromS3::nextImpl()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -1,13 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/config.h>
|
||||
|
||||
#if USE_AWS_S3
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include <IO/ConnectionTimeouts.h>
|
||||
#include <IO/HTTPCommon.h>
|
||||
#include <IO/ReadBuffer.h>
|
||||
#include <Poco/Net/HTTPBasicCredentials.h>
|
||||
#include <Poco/URI.h>
|
||||
#include <aws/s3/model/GetObjectResult.h>
|
||||
|
||||
namespace Aws::S3
|
||||
{
|
||||
class S3Client;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -15,22 +21,22 @@ namespace DB
|
||||
*/
|
||||
class ReadBufferFromS3 : public ReadBuffer
|
||||
{
|
||||
private:
|
||||
Logger * log = &Logger::get("ReadBufferFromS3");
|
||||
Aws::S3::Model::GetObjectResult read_result;
|
||||
|
||||
protected:
|
||||
Poco::URI uri;
|
||||
HTTPSessionPtr session;
|
||||
std::istream * istr; /// owned by session
|
||||
std::unique_ptr<ReadBuffer> impl;
|
||||
|
||||
RemoteHostFilter remote_host_filter;
|
||||
|
||||
public:
|
||||
explicit ReadBufferFromS3(const Poco::URI & uri_,
|
||||
const String & access_key_id_,
|
||||
const String & secret_access_key_,
|
||||
const ConnectionTimeouts & timeouts = {},
|
||||
const RemoteHostFilter & remote_host_filter_ = {});
|
||||
explicit ReadBufferFromS3(const std::shared_ptr<Aws::S3::S3Client> & client_ptr,
|
||||
const String & bucket,
|
||||
const String & key,
|
||||
size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE);
|
||||
|
||||
bool nextImpl() override;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -1,14 +1,14 @@
|
||||
#include <Common/config.h>
|
||||
|
||||
#if USE_AWS_S3
|
||||
|
||||
#include <IO/S3Common.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
|
||||
#include <iterator>
|
||||
#include <sstream>
|
||||
|
||||
#include <Poco/Base64Encoder.h>
|
||||
#include <Poco/HMACEngine.h>
|
||||
#include <Poco/SHA1Engine.h>
|
||||
#include <Poco/URI.h>
|
||||
#include <regex>
|
||||
#include <aws/s3/S3Client.h>
|
||||
#include <aws/core/auth/AWSCredentialsProvider.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -16,45 +16,94 @@ namespace DB
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_FORMAT_DATETIME;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
void S3Helper::authenticateRequest(
|
||||
Poco::Net::HTTPRequest & request,
|
||||
const String & access_key_id,
|
||||
const String & secret_access_key)
|
||||
namespace S3
|
||||
{
|
||||
/// See https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html
|
||||
|
||||
if (access_key_id.empty())
|
||||
return;
|
||||
|
||||
/// Limitations:
|
||||
/// 1. Virtual hosted-style requests are not supported (e.g. `http://johnsmith.net.s3.amazonaws.com/homepage.html`).
|
||||
/// 2. AMZ headers are not supported (TODO).
|
||||
|
||||
if (!request.has("Date"))
|
||||
ClientFactory::ClientFactory()
|
||||
{
|
||||
WriteBufferFromOwnString out;
|
||||
writeDateTimeTextRFC1123(time(nullptr), out, DateLUT::instance("UTC"));
|
||||
request.set("Date", out.str());
|
||||
aws_options = Aws::SDKOptions {};
|
||||
Aws::InitAPI(aws_options);
|
||||
}
|
||||
|
||||
String string_to_sign = request.getMethod() + "\n"
|
||||
+ request.get("Content-MD5", "") + "\n"
|
||||
+ request.get("Content-Type", "") + "\n"
|
||||
+ request.get("Date") + "\n"
|
||||
+ Poco::URI(request.getURI()).getPathAndQuery();
|
||||
ClientFactory::~ClientFactory()
|
||||
{
|
||||
Aws::ShutdownAPI(aws_options);
|
||||
}
|
||||
|
||||
Poco::HMACEngine<Poco::SHA1Engine> engine(secret_access_key);
|
||||
engine.update(string_to_sign);
|
||||
auto digest = engine.digest();
|
||||
std::ostringstream signature;
|
||||
Poco::Base64Encoder encoder(signature);
|
||||
std::copy(digest.begin(), digest.end(), std::ostream_iterator<char>(encoder));
|
||||
encoder.close();
|
||||
ClientFactory & ClientFactory::instance()
|
||||
{
|
||||
static ClientFactory ret;
|
||||
return ret;
|
||||
}
|
||||
|
||||
request.set("Authorization", "AWS " + access_key_id + ":" + signature.str());
|
||||
std::shared_ptr<Aws::S3::S3Client> ClientFactory::create(
|
||||
const String & endpoint,
|
||||
const String & access_key_id,
|
||||
const String & secret_access_key)
|
||||
{
|
||||
Aws::Client::ClientConfiguration cfg;
|
||||
if (!endpoint.empty())
|
||||
cfg.endpointOverride = endpoint;
|
||||
|
||||
auto cred_provider = std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>(access_key_id,
|
||||
secret_access_key);
|
||||
|
||||
return std::make_shared<Aws::S3::S3Client>(
|
||||
std::move(cred_provider), // Credentials provider.
|
||||
std::move(cfg), // Client configuration.
|
||||
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, // Sign policy.
|
||||
endpoint.empty() // Use virtual addressing only if endpoint is not specified.
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
URI::URI(Poco::URI & uri_)
|
||||
{
|
||||
static const std::regex BUCKET_KEY_PATTERN("([^/]+)/(.*)");
|
||||
|
||||
uri = uri_;
|
||||
|
||||
// s3://*
|
||||
if (uri.getScheme() == "s3" || uri.getScheme() == "S3")
|
||||
{
|
||||
bucket = uri.getAuthority();
|
||||
if (bucket.empty())
|
||||
throw Exception ("Invalid S3 URI: no bucket: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
const auto & path = uri.getPath();
|
||||
// s3://bucket or s3://bucket/
|
||||
if (path.length() <= 1)
|
||||
throw Exception ("Invalid S3 URI: no key: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
key = path.substr(1);
|
||||
return;
|
||||
}
|
||||
|
||||
if (uri.getHost().empty())
|
||||
throw Exception("Invalid S3 URI: no host: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
endpoint = uri.getScheme() + "://" + uri.getAuthority();
|
||||
|
||||
// Parse bucket and key from path.
|
||||
std::smatch match;
|
||||
std::regex_search(uri.getPath(), match, BUCKET_KEY_PATTERN);
|
||||
if (!match.empty())
|
||||
{
|
||||
bucket = match.str(1);
|
||||
if (bucket.empty())
|
||||
throw Exception ("Invalid S3 URI: no bucket: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
key = match.str(2);
|
||||
if (key.empty())
|
||||
throw Exception ("Invalid S3 URI: no key: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
else
|
||||
throw Exception("Invalid S3 URI: no bucket or key: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -1,19 +1,57 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/config.h>
|
||||
|
||||
#if USE_AWS_S3
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <Poco/URI.h>
|
||||
#include <aws/core/Aws.h>
|
||||
|
||||
#include <Poco/Net/HTTPRequest.h>
|
||||
namespace Aws::S3
|
||||
{
|
||||
class S3Client;
|
||||
}
|
||||
|
||||
|
||||
namespace DB
|
||||
namespace DB::S3
|
||||
{
|
||||
|
||||
namespace S3Helper
|
||||
class ClientFactory
|
||||
{
|
||||
void authenticateRequest(
|
||||
Poco::Net::HTTPRequest & request,
|
||||
const String & access_key_id,
|
||||
const String & secret_access_key);
|
||||
public:
|
||||
~ClientFactory();
|
||||
|
||||
static ClientFactory & instance();
|
||||
|
||||
std::shared_ptr<Aws::S3::S3Client> create(const String & endpoint,
|
||||
const String & access_key_id,
|
||||
const String & secret_access_key);
|
||||
|
||||
private:
|
||||
ClientFactory();
|
||||
|
||||
private:
|
||||
Aws::SDKOptions aws_options;
|
||||
};
|
||||
|
||||
/**
|
||||
* Represents S3 URI.
|
||||
*
|
||||
* The following patterns are allowed:
|
||||
* s3://bucket/key
|
||||
* http(s)://endpoint/bucket/key
|
||||
*/
|
||||
struct URI
|
||||
{
|
||||
Poco::URI uri;
|
||||
// Custom endpoint if URI scheme is not S3.
|
||||
String endpoint;
|
||||
String bucket;
|
||||
String key;
|
||||
|
||||
explicit URI (Poco::URI & uri_);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -1,22 +1,22 @@
|
||||
#include <IO/WriteBufferFromS3.h>
|
||||
#include <Common/config.h>
|
||||
|
||||
#include <IO/S3Common.h>
|
||||
#if USE_AWS_S3
|
||||
|
||||
#include <IO/WriteBufferFromS3.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
#include <Poco/DOM/AutoPtr.h>
|
||||
#include <Poco/DOM/DOMParser.h>
|
||||
#include <Poco/DOM/Document.h>
|
||||
#include <Poco/DOM/NodeList.h>
|
||||
#include <Poco/SAX/InputSource.h>
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
#include <aws/s3/S3Client.h>
|
||||
#include <aws/s3/model/CreateMultipartUploadRequest.h>
|
||||
#include <aws/s3/model/UploadPartRequest.h>
|
||||
#include <aws/s3/model/CompleteMultipartUploadRequest.h>
|
||||
|
||||
#include <utility>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
const int DEFAULT_S3_MAX_FOLLOW_PUT_REDIRECT = 2;
|
||||
|
||||
// S3 protocol does not allow to have multipart upload with more than 10000 parts.
|
||||
// In case server does not return an error on exceeding that number, we print a warning
|
||||
// because custom S3 implementation may allow relaxed requirements on that.
|
||||
@ -25,30 +25,26 @@ const int S3_WARN_MAX_PARTS = 10000;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int INCORRECT_DATA;
|
||||
extern const int S3_ERROR;
|
||||
}
|
||||
|
||||
|
||||
WriteBufferFromS3::WriteBufferFromS3(
|
||||
const Poco::URI & uri_,
|
||||
const String & access_key_id_,
|
||||
const String & secret_access_key_,
|
||||
std::shared_ptr<Aws::S3::S3Client> client_ptr_,
|
||||
const String & bucket_,
|
||||
const String & key_,
|
||||
size_t minimum_upload_part_size_,
|
||||
const ConnectionTimeouts & timeouts_,
|
||||
const RemoteHostFilter & remote_host_filter_)
|
||||
: BufferWithOwnMemory<WriteBuffer>(DBMS_DEFAULT_BUFFER_SIZE, nullptr, 0)
|
||||
, uri {uri_}
|
||||
, access_key_id {access_key_id_}
|
||||
, secret_access_key {secret_access_key_}
|
||||
size_t buffer_size_
|
||||
)
|
||||
: BufferWithOwnMemory<WriteBuffer>(buffer_size_, nullptr, 0)
|
||||
, bucket(bucket_)
|
||||
, key(key_)
|
||||
, client_ptr(std::move(client_ptr_))
|
||||
, minimum_upload_part_size {minimum_upload_part_size_}
|
||||
, timeouts {timeouts_}
|
||||
, temporary_buffer {std::make_unique<WriteBufferFromString>(buffer_string)}
|
||||
, last_part_size {0}
|
||||
, remote_host_filter(remote_host_filter_)
|
||||
{
|
||||
initiate();
|
||||
|
||||
/// FIXME: Implement rest of S3 authorization.
|
||||
}
|
||||
|
||||
|
||||
@ -98,185 +94,76 @@ WriteBufferFromS3::~WriteBufferFromS3()
|
||||
|
||||
void WriteBufferFromS3::initiate()
|
||||
{
|
||||
// See https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadInitiate.html
|
||||
Poco::Net::HTTPResponse response;
|
||||
std::unique_ptr<Poco::Net::HTTPRequest> request_ptr;
|
||||
HTTPSessionPtr session;
|
||||
std::istream * istr = nullptr; /// owned by session
|
||||
Poco::URI initiate_uri = uri;
|
||||
initiate_uri.setRawQuery("uploads");
|
||||
for (auto & param: uri.getQueryParameters())
|
||||
Aws::S3::Model::CreateMultipartUploadRequest req;
|
||||
req.SetBucket(bucket);
|
||||
req.SetKey(key);
|
||||
|
||||
auto outcome = client_ptr->CreateMultipartUpload(req);
|
||||
|
||||
if (outcome.IsSuccess())
|
||||
{
|
||||
initiate_uri.addQueryParameter(param.first, param.second);
|
||||
}
|
||||
|
||||
for (int i = 0; i < DEFAULT_S3_MAX_FOLLOW_PUT_REDIRECT; ++i)
|
||||
{
|
||||
session = makeHTTPSession(initiate_uri, timeouts);
|
||||
request_ptr = std::make_unique<Poco::Net::HTTPRequest>(Poco::Net::HTTPRequest::HTTP_POST, initiate_uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1);
|
||||
request_ptr->setHost(initiate_uri.getHost()); // use original, not resolved host name in header
|
||||
|
||||
S3Helper::authenticateRequest(*request_ptr, access_key_id, secret_access_key);
|
||||
|
||||
request_ptr->setContentLength(0);
|
||||
|
||||
LOG_TRACE((&Logger::get("WriteBufferFromS3")), "Sending request to " << initiate_uri.toString());
|
||||
|
||||
session->sendRequest(*request_ptr);
|
||||
|
||||
istr = &session->receiveResponse(response);
|
||||
|
||||
// Handle 307 Temporary Redirect in order to allow request redirection
|
||||
// See https://docs.aws.amazon.com/AmazonS3/latest/dev/Redirects.html
|
||||
if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_TEMPORARY_REDIRECT)
|
||||
break;
|
||||
|
||||
auto location_iterator = response.find("Location");
|
||||
if (location_iterator == response.end())
|
||||
break;
|
||||
|
||||
initiate_uri = location_iterator->second;
|
||||
remote_host_filter.checkURL(initiate_uri);
|
||||
}
|
||||
assertResponseIsOk(*request_ptr, response, *istr);
|
||||
|
||||
Poco::XML::InputSource src(*istr);
|
||||
Poco::XML::DOMParser parser;
|
||||
Poco::AutoPtr<Poco::XML::Document> document = parser.parse(&src);
|
||||
Poco::AutoPtr<Poco::XML::NodeList> nodes = document->getElementsByTagName("UploadId");
|
||||
if (nodes->length() != 1)
|
||||
{
|
||||
throw Exception("Incorrect XML in response, no upload id", ErrorCodes::INCORRECT_DATA);
|
||||
}
|
||||
upload_id = nodes->item(0)->innerText();
|
||||
if (upload_id.empty())
|
||||
{
|
||||
throw Exception("Incorrect XML in response, empty upload id", ErrorCodes::INCORRECT_DATA);
|
||||
upload_id = outcome.GetResult().GetUploadId();
|
||||
LOG_DEBUG(log, "Multipart upload initiated. Upload id = " + upload_id);
|
||||
}
|
||||
else
|
||||
throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
|
||||
}
|
||||
|
||||
|
||||
void WriteBufferFromS3::writePart(const String & data)
|
||||
{
|
||||
// See https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
|
||||
Poco::Net::HTTPResponse response;
|
||||
std::unique_ptr<Poco::Net::HTTPRequest> request_ptr;
|
||||
HTTPSessionPtr session;
|
||||
std::istream * istr = nullptr; /// owned by session
|
||||
Poco::URI part_uri = uri;
|
||||
part_uri.addQueryParameter("partNumber", std::to_string(part_tags.size() + 1));
|
||||
part_uri.addQueryParameter("uploadId", upload_id);
|
||||
|
||||
if (part_tags.size() == S3_WARN_MAX_PARTS)
|
||||
{
|
||||
// Don't throw exception here by ourselves but leave the decision to take by S3 server.
|
||||
LOG_WARNING(&Logger::get("WriteBufferFromS3"), "Maximum part number in S3 protocol has reached (too many parts). Server may not accept this whole upload.");
|
||||
LOG_WARNING(log, "Maximum part number in S3 protocol has reached (too many parts). Server may not accept this whole upload.");
|
||||
}
|
||||
|
||||
for (int i = 0; i < DEFAULT_S3_MAX_FOLLOW_PUT_REDIRECT; ++i)
|
||||
Aws::S3::Model::UploadPartRequest req;
|
||||
|
||||
req.SetBucket(bucket);
|
||||
req.SetKey(key);
|
||||
req.SetPartNumber(part_tags.size() + 1);
|
||||
req.SetUploadId(upload_id);
|
||||
req.SetContentLength(data.size());
|
||||
req.SetBody(std::make_shared<Aws::StringStream>(data));
|
||||
|
||||
auto outcome = client_ptr->UploadPart(req);
|
||||
|
||||
if (outcome.IsSuccess())
|
||||
{
|
||||
session = makeHTTPSession(part_uri, timeouts);
|
||||
request_ptr = std::make_unique<Poco::Net::HTTPRequest>(Poco::Net::HTTPRequest::HTTP_PUT, part_uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1);
|
||||
request_ptr->setHost(part_uri.getHost()); // use original, not resolved host name in header
|
||||
|
||||
S3Helper::authenticateRequest(*request_ptr, access_key_id, secret_access_key);
|
||||
|
||||
request_ptr->setExpectContinue(true);
|
||||
|
||||
request_ptr->setContentLength(data.size());
|
||||
|
||||
LOG_TRACE((&Logger::get("WriteBufferFromS3")), "Sending request to " << part_uri.toString());
|
||||
|
||||
std::ostream & ostr = session->sendRequest(*request_ptr);
|
||||
if (session->peekResponse(response))
|
||||
{
|
||||
// Received 100-continue.
|
||||
ostr << data;
|
||||
}
|
||||
|
||||
istr = &session->receiveResponse(response);
|
||||
|
||||
// Handle 307 Temporary Redirect in order to allow request redirection
|
||||
// See https://docs.aws.amazon.com/AmazonS3/latest/dev/Redirects.html
|
||||
if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_TEMPORARY_REDIRECT)
|
||||
break;
|
||||
|
||||
auto location_iterator = response.find("Location");
|
||||
if (location_iterator == response.end())
|
||||
break;
|
||||
|
||||
part_uri = location_iterator->second;
|
||||
auto etag = outcome.GetResult().GetETag();
|
||||
part_tags.push_back(etag);
|
||||
LOG_DEBUG(log, "Write part " + std::to_string(part_tags.size()) + " finished. Upload id = " + upload_id + ". Etag = " + etag);
|
||||
}
|
||||
assertResponseIsOk(*request_ptr, response, *istr);
|
||||
|
||||
auto etag_iterator = response.find("ETag");
|
||||
if (etag_iterator == response.end())
|
||||
{
|
||||
throw Exception("Incorrect response, no ETag", ErrorCodes::INCORRECT_DATA);
|
||||
}
|
||||
part_tags.push_back(etag_iterator->second);
|
||||
else
|
||||
throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
|
||||
}
|
||||
|
||||
|
||||
void WriteBufferFromS3::complete()
|
||||
{
|
||||
// See https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
|
||||
Poco::Net::HTTPResponse response;
|
||||
std::unique_ptr<Poco::Net::HTTPRequest> request_ptr;
|
||||
HTTPSessionPtr session;
|
||||
std::istream * istr = nullptr; /// owned by session
|
||||
Poco::URI complete_uri = uri;
|
||||
complete_uri.addQueryParameter("uploadId", upload_id);
|
||||
Aws::S3::Model::CompleteMultipartUploadRequest req;
|
||||
req.SetBucket(bucket);
|
||||
req.SetKey(key);
|
||||
req.SetUploadId(upload_id);
|
||||
|
||||
String data;
|
||||
WriteBufferFromString buffer(data);
|
||||
writeString("<CompleteMultipartUpload>", buffer);
|
||||
Aws::S3::Model::CompletedMultipartUpload multipart_upload;
|
||||
for (size_t i = 0; i < part_tags.size(); ++i)
|
||||
{
|
||||
writeString("<Part><PartNumber>", buffer);
|
||||
writeIntText(i + 1, buffer);
|
||||
writeString("</PartNumber><ETag>", buffer);
|
||||
writeString(part_tags[i], buffer);
|
||||
writeString("</ETag></Part>", buffer);
|
||||
Aws::S3::Model::CompletedPart part;
|
||||
multipart_upload.AddParts(part.WithETag(part_tags[i]).WithPartNumber(i + 1));
|
||||
}
|
||||
writeString("</CompleteMultipartUpload>", buffer);
|
||||
buffer.finish();
|
||||
|
||||
for (int i = 0; i < DEFAULT_S3_MAX_FOLLOW_PUT_REDIRECT; ++i)
|
||||
{
|
||||
session = makeHTTPSession(complete_uri, timeouts);
|
||||
request_ptr = std::make_unique<Poco::Net::HTTPRequest>(Poco::Net::HTTPRequest::HTTP_POST, complete_uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1);
|
||||
request_ptr->setHost(complete_uri.getHost()); // use original, not resolved host name in header
|
||||
req.SetMultipartUpload(multipart_upload);
|
||||
|
||||
S3Helper::authenticateRequest(*request_ptr, access_key_id, secret_access_key);
|
||||
auto outcome = client_ptr->CompleteMultipartUpload(req);
|
||||
|
||||
request_ptr->setExpectContinue(true);
|
||||
|
||||
request_ptr->setContentLength(data.size());
|
||||
|
||||
LOG_TRACE((&Logger::get("WriteBufferFromS3")), "Sending request to " << complete_uri.toString());
|
||||
|
||||
std::ostream & ostr = session->sendRequest(*request_ptr);
|
||||
if (session->peekResponse(response))
|
||||
{
|
||||
// Received 100-continue.
|
||||
ostr << data;
|
||||
}
|
||||
|
||||
istr = &session->receiveResponse(response);
|
||||
|
||||
// Handle 307 Temporary Redirect in order to allow request redirection
|
||||
// See https://docs.aws.amazon.com/AmazonS3/latest/dev/Redirects.html
|
||||
if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_TEMPORARY_REDIRECT)
|
||||
break;
|
||||
|
||||
auto location_iterator = response.find("Location");
|
||||
if (location_iterator == response.end())
|
||||
break;
|
||||
|
||||
complete_uri = location_iterator->second;
|
||||
}
|
||||
assertResponseIsOk(*request_ptr, response, *istr);
|
||||
if (outcome.IsSuccess())
|
||||
LOG_DEBUG(log, "Multipart upload completed. Upload_id = " + upload_id);
|
||||
else
|
||||
throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -1,17 +1,21 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/config.h>
|
||||
|
||||
#if USE_AWS_S3
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <Core/Types.h>
|
||||
#include <IO/ConnectionTimeouts.h>
|
||||
#include <IO/HTTPCommon.h>
|
||||
#include <IO/BufferWithOwnMemory.h>
|
||||
#include <IO/WriteBuffer.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <Poco/Net/HTTPBasicCredentials.h>
|
||||
#include <Poco/Net/HTTPRequest.h>
|
||||
#include <Poco/URI.h>
|
||||
|
||||
namespace Aws::S3
|
||||
{
|
||||
class S3Client;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -20,28 +24,27 @@ namespace DB
|
||||
class WriteBufferFromS3 : public BufferWithOwnMemory<WriteBuffer>
|
||||
{
|
||||
private:
|
||||
Poco::URI uri;
|
||||
String access_key_id;
|
||||
String secret_access_key;
|
||||
String bucket;
|
||||
String key;
|
||||
std::shared_ptr<Aws::S3::S3Client> client_ptr;
|
||||
size_t minimum_upload_part_size;
|
||||
ConnectionTimeouts timeouts;
|
||||
String buffer_string;
|
||||
std::unique_ptr<WriteBufferFromString> temporary_buffer;
|
||||
size_t last_part_size;
|
||||
RemoteHostFilter remote_host_filter;
|
||||
|
||||
/// Upload in S3 is made in parts.
|
||||
/// We initiate upload, then upload each part and get ETag as a response, and then finish upload with listing all our parts.
|
||||
String upload_id;
|
||||
std::vector<String> part_tags;
|
||||
|
||||
Logger * log = &Logger::get("WriteBufferFromS3");
|
||||
|
||||
public:
|
||||
explicit WriteBufferFromS3(const Poco::URI & uri,
|
||||
const String & access_key_id,
|
||||
const String & secret_access_key,
|
||||
size_t minimum_upload_part_size_,
|
||||
const ConnectionTimeouts & timeouts = {},
|
||||
const RemoteHostFilter & remote_host_filter_ = {});
|
||||
explicit WriteBufferFromS3(std::shared_ptr<Aws::S3::S3Client> client_ptr_,
|
||||
const String & bucket_,
|
||||
const String & key_,
|
||||
size_t minimum_upload_part_size_,
|
||||
size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE);
|
||||
|
||||
void nextImpl() override;
|
||||
|
||||
@ -57,3 +60,5 @@ private:
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -1,3 +1,8 @@
|
||||
#include <Common/config.h>
|
||||
|
||||
#if USE_AWS_S3
|
||||
|
||||
#include <IO/S3Common.h>
|
||||
#include <Storages/StorageFactory.h>
|
||||
#include <Storages/StorageS3.h>
|
||||
|
||||
@ -16,7 +21,7 @@
|
||||
#include <DataStreams/IBlockInputStream.h>
|
||||
#include <DataStreams/AddingDefaultsBlockInputStream.h>
|
||||
|
||||
#include <Poco/Net/HTTPRequest.h>
|
||||
#include <aws/s3/S3Client.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -26,24 +31,25 @@ namespace ErrorCodes
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
}
|
||||
|
||||
|
||||
namespace
|
||||
{
|
||||
class StorageS3BlockInputStream : public IBlockInputStream
|
||||
{
|
||||
public:
|
||||
StorageS3BlockInputStream(const Poco::URI & uri,
|
||||
const String & access_key_id,
|
||||
const String & secret_access_key,
|
||||
StorageS3BlockInputStream(
|
||||
const String & format,
|
||||
const String & name_,
|
||||
const Block & sample_block,
|
||||
const Context & context,
|
||||
UInt64 max_block_size,
|
||||
const ConnectionTimeouts & timeouts,
|
||||
const CompressionMethod compression_method)
|
||||
const CompressionMethod compression_method,
|
||||
const std::shared_ptr<Aws::S3::S3Client> & client,
|
||||
const String & bucket,
|
||||
const String & key)
|
||||
: name(name_)
|
||||
{
|
||||
read_buf = getReadBuffer<ReadBufferFromS3>(compression_method, uri, access_key_id, secret_access_key, timeouts, context.getRemoteHostFilter());
|
||||
read_buf = getReadBuffer<ReadBufferFromS3>(compression_method, client, bucket, key);
|
||||
reader = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size);
|
||||
}
|
||||
|
||||
@ -81,25 +87,18 @@ namespace
|
||||
class StorageS3BlockOutputStream : public IBlockOutputStream
|
||||
{
|
||||
public:
|
||||
StorageS3BlockOutputStream(const Poco::URI & uri,
|
||||
const String & access_key_id,
|
||||
const String & secret_access_key,
|
||||
StorageS3BlockOutputStream(
|
||||
const String & format,
|
||||
UInt64 min_upload_part_size,
|
||||
const Block & sample_block_,
|
||||
const Context & context,
|
||||
const ConnectionTimeouts & timeouts,
|
||||
const CompressionMethod compression_method)
|
||||
const CompressionMethod compression_method,
|
||||
const std::shared_ptr<Aws::S3::S3Client> & client,
|
||||
const String & bucket,
|
||||
const String & key)
|
||||
: sample_block(sample_block_)
|
||||
{
|
||||
write_buf = getWriteBuffer<WriteBufferFromS3>(
|
||||
compression_method,
|
||||
uri,
|
||||
access_key_id,
|
||||
secret_access_key,
|
||||
min_upload_part_size,
|
||||
timeouts,
|
||||
context.getRemoteHostFilter());
|
||||
write_buf = getWriteBuffer<WriteBufferFromS3>(compression_method, client, bucket, key, min_upload_part_size);
|
||||
writer = FormatFactory::instance().getOutput(format, *write_buf, sample_block, context);
|
||||
}
|
||||
|
||||
@ -133,8 +132,7 @@ namespace
|
||||
}
|
||||
|
||||
|
||||
StorageS3::StorageS3(
|
||||
const Poco::URI & uri_,
|
||||
StorageS3::StorageS3(const S3::URI & uri_,
|
||||
const String & access_key_id_,
|
||||
const String & secret_access_key_,
|
||||
const std::string & database_name_,
|
||||
@ -147,16 +145,15 @@ StorageS3::StorageS3(
|
||||
const String & compression_method_ = "")
|
||||
: IStorage(columns_)
|
||||
, uri(uri_)
|
||||
, access_key_id(access_key_id_)
|
||||
, secret_access_key(secret_access_key_)
|
||||
, context_global(context_)
|
||||
, format_name(format_name_)
|
||||
, database_name(database_name_)
|
||||
, table_name(table_name_)
|
||||
, min_upload_part_size(min_upload_part_size_)
|
||||
, compression_method(compression_method_)
|
||||
, client(S3::ClientFactory::instance().create(uri_.endpoint, access_key_id_, secret_access_key_))
|
||||
{
|
||||
context_global.getRemoteHostFilter().checkURL(uri_);
|
||||
context_global.getRemoteHostFilter().checkURL(uri_.uri);
|
||||
setColumns(columns_);
|
||||
setConstraints(constraints_);
|
||||
}
|
||||
@ -171,16 +168,15 @@ BlockInputStreams StorageS3::read(
|
||||
unsigned /*num_streams*/)
|
||||
{
|
||||
BlockInputStreamPtr block_input = std::make_shared<StorageS3BlockInputStream>(
|
||||
uri,
|
||||
access_key_id,
|
||||
secret_access_key,
|
||||
format_name,
|
||||
getName(),
|
||||
getHeaderBlock(column_names),
|
||||
context,
|
||||
max_block_size,
|
||||
ConnectionTimeouts::getHTTPTimeouts(context),
|
||||
IStorage::chooseCompressionMethod(uri.toString(), compression_method));
|
||||
IStorage::chooseCompressionMethod(uri.endpoint, compression_method),
|
||||
client,
|
||||
uri.bucket,
|
||||
uri.key);
|
||||
|
||||
auto column_defaults = getColumns().getDefaults();
|
||||
if (column_defaults.empty())
|
||||
@ -197,15 +193,9 @@ void StorageS3::rename(const String & /*new_path_to_db*/, const String & new_dat
|
||||
BlockOutputStreamPtr StorageS3::write(const ASTPtr & /*query*/, const Context & /*context*/)
|
||||
{
|
||||
return std::make_shared<StorageS3BlockOutputStream>(
|
||||
uri,
|
||||
access_key_id,
|
||||
secret_access_key,
|
||||
format_name,
|
||||
min_upload_part_size,
|
||||
getSampleBlock(),
|
||||
context_global,
|
||||
ConnectionTimeouts::getHTTPTimeouts(context_global),
|
||||
IStorage::chooseCompressionMethod(uri.toString(), compression_method));
|
||||
format_name, min_upload_part_size, getSampleBlock(), context_global,
|
||||
IStorage::chooseCompressionMethod(uri.endpoint, compression_method),
|
||||
client, uri.bucket, uri.key);
|
||||
}
|
||||
|
||||
void registerStorageS3(StorageFactory & factory)
|
||||
@ -222,7 +212,8 @@ void registerStorageS3(StorageFactory & factory)
|
||||
engine_args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[i], args.local_context);
|
||||
|
||||
String url = engine_args[0]->as<ASTLiteral &>().value.safeGet<String>();
|
||||
Poco::URI uri(url);
|
||||
Poco::URI uri (url);
|
||||
S3::URI s3_uri (uri);
|
||||
|
||||
String format_name = engine_args[engine_args.size() - 1]->as<ASTLiteral &>().value.safeGet<String>();
|
||||
|
||||
@ -242,7 +233,10 @@ void registerStorageS3(StorageFactory & factory)
|
||||
else
|
||||
compression_method = "auto";
|
||||
|
||||
return StorageS3::create(uri, access_key_id, secret_access_key, args.database_name, args.table_name, format_name, min_upload_part_size, args.columns, args.constraints, args.context);
|
||||
return StorageS3::create(s3_uri, access_key_id, secret_access_key, args.database_name, args.table_name, format_name, min_upload_part_size, args.columns, args.constraints, args.context);
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -1,13 +1,22 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/config.h>
|
||||
|
||||
#if USE_AWS_S3
|
||||
|
||||
#include <Storages/IStorage.h>
|
||||
#include <Poco/URI.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <ext/shared_ptr_helper.h>
|
||||
|
||||
namespace Aws::S3
|
||||
{
|
||||
class S3Client;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/**
|
||||
* This class represents table engine for external S3 urls.
|
||||
* It sends HTTP GET to server when select is called and
|
||||
@ -16,8 +25,7 @@ namespace DB
|
||||
class StorageS3 : public ext::shared_ptr_helper<StorageS3>, public IStorage
|
||||
{
|
||||
public:
|
||||
StorageS3(
|
||||
const Poco::URI & uri_,
|
||||
StorageS3(const S3::URI & uri,
|
||||
const String & access_key_id,
|
||||
const String & secret_access_key,
|
||||
const String & database_name_,
|
||||
@ -57,9 +65,7 @@ public:
|
||||
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
|
||||
|
||||
private:
|
||||
Poco::URI uri;
|
||||
String access_key_id;
|
||||
String secret_access_key;
|
||||
S3::URI uri;
|
||||
const Context & context_global;
|
||||
|
||||
String format_name;
|
||||
@ -67,6 +73,9 @@ private:
|
||||
String table_name;
|
||||
UInt64 min_upload_part_size;
|
||||
String compression_method;
|
||||
std::shared_ptr<Aws::S3::S3Client> client;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -19,7 +19,6 @@ void registerStorageDistributed(StorageFactory & factory);
|
||||
void registerStorageMemory(StorageFactory & factory);
|
||||
void registerStorageFile(StorageFactory & factory);
|
||||
void registerStorageURL(StorageFactory & factory);
|
||||
void registerStorageS3(StorageFactory & factory);
|
||||
void registerStorageDictionary(StorageFactory & factory);
|
||||
void registerStorageSet(StorageFactory & factory);
|
||||
void registerStorageJoin(StorageFactory & factory);
|
||||
@ -27,6 +26,10 @@ void registerStorageView(StorageFactory & factory);
|
||||
void registerStorageMaterializedView(StorageFactory & factory);
|
||||
void registerStorageLiveView(StorageFactory & factory);
|
||||
|
||||
#if USE_AWS_S3
|
||||
void registerStorageS3(StorageFactory & factory);
|
||||
#endif
|
||||
|
||||
#if USE_HDFS
|
||||
void registerStorageHDFS(StorageFactory & factory);
|
||||
#endif
|
||||
@ -61,7 +64,6 @@ void registerStorages()
|
||||
registerStorageMemory(factory);
|
||||
registerStorageFile(factory);
|
||||
registerStorageURL(factory);
|
||||
registerStorageS3(factory);
|
||||
registerStorageDictionary(factory);
|
||||
registerStorageSet(factory);
|
||||
registerStorageJoin(factory);
|
||||
@ -69,6 +71,10 @@ void registerStorages()
|
||||
registerStorageMaterializedView(factory);
|
||||
registerStorageLiveView(factory);
|
||||
|
||||
#if USE_AWS_S3
|
||||
registerStorageS3(factory);
|
||||
#endif
|
||||
|
||||
#if USE_HDFS
|
||||
registerStorageHDFS(factory);
|
||||
#endif
|
||||
|
||||
@ -1,10 +1,14 @@
|
||||
#include <Common/config.h>
|
||||
|
||||
#if USE_AWS_S3
|
||||
|
||||
#include <IO/S3Common.h>
|
||||
#include <Storages/StorageS3.h>
|
||||
#include <Interpreters/evaluateConstantExpression.h>
|
||||
#include <TableFunctions/TableFunctionFactory.h>
|
||||
#include <TableFunctions/TableFunctionS3.h>
|
||||
#include <TableFunctions/parseColumnsListForTableFunction.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Poco/URI.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -76,9 +80,11 @@ StoragePtr TableFunctionS3::getStorage(
|
||||
const std::string & table_name,
|
||||
const String & compression_method) const
|
||||
{
|
||||
Poco::URI uri(source);
|
||||
Poco::URI uri (source);
|
||||
S3::URI s3_uri (uri);
|
||||
|
||||
UInt64 min_upload_part_size = global_context.getSettingsRef().s3_min_upload_part_size;
|
||||
return StorageS3::create(uri, access_key_id, secret_access_key, getDatabaseName(), table_name, format, min_upload_part_size, columns, ConstraintsDescription{}, global_context, compression_method);
|
||||
return StorageS3::create(s3_uri, access_key_id, secret_access_key, getDatabaseName(), table_name, format, min_upload_part_size, columns, ConstraintsDescription{}, global_context, compression_method);
|
||||
}
|
||||
|
||||
void registerTableFunctionS3(TableFunctionFactory & factory)
|
||||
@ -87,3 +93,5 @@ void registerTableFunctionS3(TableFunctionFactory & factory)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -1,5 +1,9 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/config.h>
|
||||
|
||||
#if USE_AWS_S3
|
||||
|
||||
#include <TableFunctions/ITableFunction.h>
|
||||
|
||||
|
||||
@ -37,3 +41,5 @@ private:
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -11,11 +11,14 @@ void registerTableFunctionMerge(TableFunctionFactory & factory);
|
||||
void registerTableFunctionRemote(TableFunctionFactory & factory);
|
||||
void registerTableFunctionNumbers(TableFunctionFactory & factory);
|
||||
void registerTableFunctionFile(TableFunctionFactory & factory);
|
||||
void registerTableFunctionS3(TableFunctionFactory & factory);
|
||||
void registerTableFunctionURL(TableFunctionFactory & factory);
|
||||
void registerTableFunctionValues(TableFunctionFactory & factory);
|
||||
void registerTableFunctionInput(TableFunctionFactory & factory);
|
||||
|
||||
#if USE_AWS_S3
|
||||
void registerTableFunctionS3(TableFunctionFactory & factory);
|
||||
#endif
|
||||
|
||||
#if USE_HDFS
|
||||
void registerTableFunctionHDFS(TableFunctionFactory & factory);
|
||||
#endif
|
||||
@ -39,11 +42,14 @@ void registerTableFunctions()
|
||||
registerTableFunctionRemote(factory);
|
||||
registerTableFunctionNumbers(factory);
|
||||
registerTableFunctionFile(factory);
|
||||
registerTableFunctionS3(factory);
|
||||
registerTableFunctionURL(factory);
|
||||
registerTableFunctionValues(factory);
|
||||
registerTableFunctionInput(factory);
|
||||
|
||||
#if USE_AWS_S3
|
||||
registerTableFunctionS3(factory);
|
||||
#endif
|
||||
|
||||
#if USE_HDFS
|
||||
registerTableFunctionHDFS(factory);
|
||||
#endif
|
||||
|
||||
@ -112,9 +112,9 @@ def run_query(instance, query, stdin=None, settings=None):
|
||||
|
||||
# Test simple put.
|
||||
@pytest.mark.parametrize("maybe_auth,positive", [
|
||||
("",True),
|
||||
("'minio','minio123',",True),
|
||||
("'wrongid','wrongkey',",False)
|
||||
("", True),
|
||||
("'minio','minio123',", True),
|
||||
("'wrongid','wrongkey',", False)
|
||||
])
|
||||
def test_put(cluster, maybe_auth, positive):
|
||||
# type: (ClickHouseCluster) -> None
|
||||
@ -131,7 +131,8 @@ def test_put(cluster, maybe_auth, positive):
|
||||
try:
|
||||
run_query(instance, put_query)
|
||||
except helpers.client.QueryRuntimeException:
|
||||
assert not positive
|
||||
if positive:
|
||||
raise
|
||||
else:
|
||||
assert positive
|
||||
assert values_csv == get_s3_file_content(cluster, bucket, filename)
|
||||
@ -139,9 +140,9 @@ def test_put(cluster, maybe_auth, positive):
|
||||
|
||||
# Test put values in CSV format.
|
||||
@pytest.mark.parametrize("maybe_auth,positive", [
|
||||
("",True),
|
||||
("'minio','minio123',",True),
|
||||
("'wrongid','wrongkey',",False)
|
||||
("", True),
|
||||
("'minio','minio123',", True),
|
||||
("'wrongid','wrongkey',", False)
|
||||
])
|
||||
def test_put_csv(cluster, maybe_auth, positive):
|
||||
# type: (ClickHouseCluster) -> None
|
||||
@ -157,7 +158,8 @@ def test_put_csv(cluster, maybe_auth, positive):
|
||||
try:
|
||||
run_query(instance, put_query, stdin=csv_data)
|
||||
except helpers.client.QueryRuntimeException:
|
||||
assert not positive
|
||||
if positive:
|
||||
raise
|
||||
else:
|
||||
assert positive
|
||||
assert csv_data == get_s3_file_content(cluster, bucket, filename)
|
||||
@ -192,9 +194,9 @@ def test_put_get_with_redirect(cluster):
|
||||
|
||||
# Test multipart put.
|
||||
@pytest.mark.parametrize("maybe_auth,positive", [
|
||||
("",True),
|
||||
("'minio','minio123',",True),
|
||||
("'wrongid','wrongkey',",False)
|
||||
("", True),
|
||||
# ("'minio','minio123',",True), Redirect with credentials not working with nginx.
|
||||
("'wrongid','wrongkey',", False)
|
||||
])
|
||||
def test_multipart_put(cluster, maybe_auth, positive):
|
||||
# type: (ClickHouseCluster) -> None
|
||||
@ -223,14 +225,15 @@ def test_multipart_put(cluster, maybe_auth, positive):
|
||||
try:
|
||||
run_query(instance, put_query, stdin=csv_data, settings={'s3_min_upload_part_size': min_part_size_bytes})
|
||||
except helpers.client.QueryRuntimeException:
|
||||
assert not positive
|
||||
if positive:
|
||||
raise
|
||||
else:
|
||||
assert positive
|
||||
|
||||
# Use Nginx access logs to count number of parts uploaded to Minio.
|
||||
nginx_logs = get_nginx_access_logs()
|
||||
uploaded_parts = filter(lambda log_line: log_line.find(filename) >= 0 and log_line.find("PUT") >= 0, nginx_logs)
|
||||
assert uploaded_parts > 1
|
||||
assert len(uploaded_parts) > 1
|
||||
|
||||
assert csv_data == get_s3_file_content(cluster, bucket, filename)
|
||||
|
||||
@ -239,9 +242,23 @@ def test_remote_host_filter(cluster):
|
||||
instance = cluster.instances["restricted_dummy"]
|
||||
format = "column1 UInt32, column2 UInt32, column3 UInt32"
|
||||
|
||||
query = "select *, column1*column2*column3 from s3('http://{}:{}/', 'CSV', '{}')".format("invalid_host", cluster.minio_redirect_port, format)
|
||||
query = "select *, column1*column2*column3 from s3('http://{}:{}/{}/test.csv', 'CSV', '{}')".format(
|
||||
"invalid_host", cluster.minio_port, cluster.minio_bucket, format)
|
||||
assert "not allowed in config.xml" in instance.query_and_get_error(query)
|
||||
|
||||
other_values = "(1, 1, 1), (1, 1, 1), (11, 11, 11)"
|
||||
query = "insert into table function s3('http://{}:{}/{}/test.csv', 'CSV', '{}') values {}".format("invalid_host", cluster.minio_port, cluster.minio_bucket, format, other_values)
|
||||
query = "insert into table function s3('http://{}:{}/{}/test.csv', 'CSV', '{}') values {}".format(
|
||||
"invalid_host", cluster.minio_port, cluster.minio_bucket, format, other_values)
|
||||
assert "not allowed in config.xml" in instance.query_and_get_error(query)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("s3_storage_args", [
|
||||
"''", # 1 arguments
|
||||
"'','','','','',''" # 6 arguments
|
||||
])
|
||||
def test_wrong_s3_syntax(cluster, s3_storage_args):
|
||||
instance = cluster.instances["dummy"] # type: ClickHouseInstance
|
||||
expected_err_msg = "Code: 42" # NUMBER_OF_ARGUMENTS_DOESNT_MATCH
|
||||
|
||||
query = "create table test_table_s3_syntax (id UInt32) ENGINE = S3({})".format(s3_storage_args)
|
||||
assert expected_err_msg in instance.query_and_get_error(query)
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
drop table if exists test_table_s3_syntax
|
||||
;
|
||||
create table test_table_s3_syntax (id UInt32) ENGINE = S3('')
|
||||
; -- { serverError 42 }
|
||||
create table test_table_s3_syntax (id UInt32) ENGINE = S3('','','','','','')
|
||||
; -- { serverError 42 }
|
||||
drop table if exists test_table_s3_syntax
|
||||
;
|
||||
Loading…
Reference in New Issue
Block a user