diff --git a/.gitmodules b/.gitmodules index 0805b6d5492..406e8a7e11e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -104,13 +104,13 @@ url = https://github.com/ClickHouse/aws-sdk-cpp.git [submodule "aws-c-event-stream"] path = contrib/aws-c-event-stream - url = https://github.com/ClickHouse/aws-c-event-stream.git + url = https://github.com/awslabs/aws-c-event-stream.git [submodule "aws-c-common"] path = contrib/aws-c-common url = https://github.com/ClickHouse/aws-c-common.git [submodule "aws-checksums"] path = contrib/aws-checksums - url = https://github.com/ClickHouse/aws-checksums.git + url = https://github.com/awslabs/aws-checksums.git [submodule "contrib/curl"] path = contrib/curl url = https://github.com/curl/curl.git @@ -294,3 +294,33 @@ [submodule "contrib/libdivide"] path = contrib/libdivide url = https://github.com/ridiculousfish/libdivide.git +[submodule "contrib/aws-crt-cpp"] + path = contrib/aws-crt-cpp + url = https://github.com/ClickHouse/aws-crt-cpp.git +[submodule "contrib/aws-c-io"] + path = contrib/aws-c-io + url = https://github.com/ClickHouse/aws-c-io.git +[submodule "contrib/aws-c-mqtt"] + path = contrib/aws-c-mqtt + url = https://github.com/awslabs/aws-c-mqtt.git +[submodule "contrib/aws-c-auth"] + path = contrib/aws-c-auth + url = https://github.com/awslabs/aws-c-auth.git +[submodule "contrib/aws-c-cal"] + path = contrib/aws-c-cal + url = https://github.com/ClickHouse/aws-c-cal.git +[submodule "contrib/aws-c-sdkutils"] + path = contrib/aws-c-sdkutils + url = https://github.com/awslabs/aws-c-sdkutils.git +[submodule "contrib/aws-c-http"] + path = contrib/aws-c-http + url = https://github.com/awslabs/aws-c-http.git +[submodule "contrib/aws-c-s3"] + path = contrib/aws-c-s3 + url = https://github.com/awslabs/aws-c-s3.git +[submodule "contrib/aws-c-compression"] + path = contrib/aws-c-compression + url = https://github.com/awslabs/aws-c-compression.git +[submodule "contrib/aws-s2n-tls"] + path = contrib/aws-s2n-tls + url = https://github.com/aws/s2n-tls.git diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 6f80059498e..13c4722e149 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -115,12 +115,25 @@ endif() add_contrib (llvm-project-cmake llvm-project) add_contrib (libfuzzer-cmake llvm-project) add_contrib (libxml2-cmake libxml2) -add_contrib (aws-s3-cmake + +add_contrib (aws-cmake aws + aws-c-auth + aws-c-cal aws-c-common + aws-c-compression aws-c-event-stream + aws-c-http + aws-c-io + aws-c-mqtt + aws-c-s3 + aws-c-sdkutils + aws-s2n-tls aws-checksums + aws-crt-cpp + aws-cmake ) + add_contrib (base64-cmake base64) add_contrib (simdjson-cmake simdjson) add_contrib (rapidjson-cmake rapidjson) diff --git a/contrib/aws b/contrib/aws index 00b03604543..4a12641211d 160000 --- a/contrib/aws +++ b/contrib/aws @@ -1 +1 @@ -Subproject commit 00b03604543367d7e310cb0993973fdcb723ea79 +Subproject commit 4a12641211d4dbc8e2fdb2dd0f1eea0927db9252 diff --git a/contrib/aws-c-auth b/contrib/aws-c-auth new file mode 160000 index 00000000000..30df6c407e2 --- /dev/null +++ b/contrib/aws-c-auth @@ -0,0 +1 @@ +Subproject commit 30df6c407e2df43bd244e2c34c9b4a4b87372bfb diff --git a/contrib/aws-c-cal b/contrib/aws-c-cal new file mode 160000 index 00000000000..85dd7664b78 --- /dev/null +++ b/contrib/aws-c-cal @@ -0,0 +1 @@ +Subproject commit 85dd7664b786a389c6fb1a6f031ab4bb2282133d diff --git a/contrib/aws-c-common b/contrib/aws-c-common index 736a82d1697..324fd1d973c 160000 --- a/contrib/aws-c-common +++ b/contrib/aws-c-common @@ -1 +1 @@ -Subproject commit 736a82d1697c108b04a277e66438a7f4e19b6857 +Subproject commit 324fd1d973ccb25c813aa747bf1759cfde5121c5 diff --git a/contrib/aws-c-compression b/contrib/aws-c-compression new file mode 160000 index 00000000000..b517b7decd0 --- /dev/null +++ b/contrib/aws-c-compression @@ -0,0 +1 @@ +Subproject commit b517b7decd0dac30be2162f5186c250221c53aff diff --git a/contrib/aws-c-event-stream b/contrib/aws-c-event-stream index 3bc33662f9c..39bfa94a14b 160000 --- a/contrib/aws-c-event-stream +++ b/contrib/aws-c-event-stream @@ -1 +1 @@ -Subproject commit 3bc33662f9ccff4f4cbcf9509cc78c26e022fde0 +Subproject commit 39bfa94a14b7126bf0c1330286ef8db452d87e66 diff --git a/contrib/aws-c-http b/contrib/aws-c-http new file mode 160000 index 00000000000..2c5a2a7d555 --- /dev/null +++ b/contrib/aws-c-http @@ -0,0 +1 @@ +Subproject commit 2c5a2a7d5556600b9782ffa6c9d7e09964df1abc diff --git a/contrib/aws-c-io b/contrib/aws-c-io new file mode 160000 index 00000000000..5d32c453560 --- /dev/null +++ b/contrib/aws-c-io @@ -0,0 +1 @@ +Subproject commit 5d32c453560d0823df521a686bf7fbacde7f9be3 diff --git a/contrib/aws-c-mqtt b/contrib/aws-c-mqtt new file mode 160000 index 00000000000..882c689561a --- /dev/null +++ b/contrib/aws-c-mqtt @@ -0,0 +1 @@ +Subproject commit 882c689561a3db1466330ccfe3b63637e0a575d3 diff --git a/contrib/aws-c-s3 b/contrib/aws-c-s3 new file mode 160000 index 00000000000..a41255ece72 --- /dev/null +++ b/contrib/aws-c-s3 @@ -0,0 +1 @@ +Subproject commit a41255ece72a7c887bba7f9d998ca3e14f4c8a1b diff --git a/contrib/aws-c-sdkutils b/contrib/aws-c-sdkutils new file mode 160000 index 00000000000..25bf5cf225f --- /dev/null +++ b/contrib/aws-c-sdkutils @@ -0,0 +1 @@ +Subproject commit 25bf5cf225f977c3accc6a05a0a7a181ef2a4a30 diff --git a/contrib/aws-checksums b/contrib/aws-checksums index 519d6d90938..48e7c0e0147 160000 --- a/contrib/aws-checksums +++ b/contrib/aws-checksums @@ -1 +1 @@ -Subproject commit 519d6d9093819b6cf89ffff589a27ef8f83d0f65 +Subproject commit 48e7c0e01479232f225c8044d76c84e74192889d diff --git a/contrib/aws-cmake/AwsFeatureTests.cmake b/contrib/aws-cmake/AwsFeatureTests.cmake new file mode 100644 index 00000000000..54727e08d6b --- /dev/null +++ b/contrib/aws-cmake/AwsFeatureTests.cmake @@ -0,0 +1,114 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +include(CheckCSourceRuns) + +option(USE_CPU_EXTENSIONS "Whenever possible, use functions optimized for CPUs with specific extensions (ex: SSE, AVX)." ON) + +# In the current (11/2/21) state of mingw64, the packaged gcc is not capable of emitting properly aligned avx2 instructions under certain circumstances. +# This leads to crashes for windows builds using mingw64 when invoking the avx2-enabled versions of certain functions. Until we can find a better +# work-around, disable avx2 (and all other extensions) in mingw builds. +# +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412 +# +if (MINGW) + message(STATUS "MINGW detected! Disabling avx2 and other CPU extensions") + set(USE_CPU_EXTENSIONS OFF) +endif() + +if(NOT CMAKE_CROSSCOMPILING) + check_c_source_runs(" + #include + bool foo(int a, int b, int *c) { + return __builtin_mul_overflow(a, b, c); + } + + int main() { + int out; + if (foo(1, 2, &out)) { + return 0; + } + + return 0; + }" AWS_HAVE_GCC_OVERFLOW_MATH_EXTENSIONS) + + if (USE_CPU_EXTENSIONS) + check_c_source_runs(" + int main() { + int foo = 42; + _mulx_u32(1, 2, &foo); + return foo != 2; + }" AWS_HAVE_MSVC_MULX) + endif() + +endif() + +check_c_source_compiles(" + #include + #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + int main() { + return 0; + } + #else + it's not windows desktop + #endif +" AWS_HAVE_WINAPI_DESKTOP) + +check_c_source_compiles(" + int main() { +#if !(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) +# error \"not intel\" +#endif + return 0; + } +" AWS_ARCH_INTEL) + +check_c_source_compiles(" + int main() { +#if !(defined(__aarch64__) || defined(_M_ARM64)) +# error \"not arm64\" +#endif + return 0; + } +" AWS_ARCH_ARM64) + +check_c_source_compiles(" + int main() { +#if !(defined(__arm__) || defined(_M_ARM)) +# error \"not arm\" +#endif + return 0; + } +" AWS_ARCH_ARM32) + +check_c_source_compiles(" +int main() { + int foo = 42, bar = 24; + __asm__ __volatile__(\"\":\"=r\"(foo):\"r\"(bar):\"memory\"); +}" AWS_HAVE_GCC_INLINE_ASM) + +check_c_source_compiles(" +#include +int main() { +#ifdef __linux__ + getauxval(AT_HWCAP); + getauxval(AT_HWCAP2); +#endif + return 0; +}" AWS_HAVE_AUXV) + +string(REGEX MATCH "^(aarch64|arm)" ARM_CPU "${CMAKE_SYSTEM_PROCESSOR}") +if(NOT LEGACY_COMPILER_SUPPORT OR ARM_CPU) + check_c_source_compiles(" + #include + int main() { + backtrace(NULL, 0); + return 0; + }" AWS_HAVE_EXECINFO) +endif() + +check_c_source_compiles(" +#include +int main() { + return 1; +}" AWS_HAVE_LINUX_IF_LINK_H) diff --git a/contrib/aws-cmake/AwsSIMD.cmake b/contrib/aws-cmake/AwsSIMD.cmake new file mode 100644 index 00000000000..bd6f4064e78 --- /dev/null +++ b/contrib/aws-cmake/AwsSIMD.cmake @@ -0,0 +1,74 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +include(CheckCCompilerFlag) +include(CheckIncludeFile) + +if (USE_CPU_EXTENSIONS) + if (MSVC) + check_c_compiler_flag("/arch:AVX2" HAVE_M_AVX2_FLAG) + if (HAVE_M_AVX2_FLAG) + set(AVX2_CFLAGS "/arch:AVX2") + endif() + else() + check_c_compiler_flag(-mavx2 HAVE_M_AVX2_FLAG) + if (HAVE_M_AVX2_FLAG) + set(AVX2_CFLAGS "-mavx -mavx2") + endif() + endif() + + + cmake_push_check_state() + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${AVX2_CFLAGS}") + + check_c_source_compiles(" + #include + #include + #include + + int main() { + __m256i vec; + memset(&vec, 0, sizeof(vec)); + + _mm256_shuffle_epi8(vec, vec); + _mm256_set_epi32(1,2,3,4,5,6,7,8); + _mm256_permutevar8x32_epi32(vec, vec); + + return 0; + }" HAVE_AVX2_INTRINSICS) + + check_c_source_compiles(" + #include + #include + + int main() { + __m256i vec; + memset(&vec, 0, sizeof(vec)); + return (int)_mm256_extract_epi64(vec, 2); + }" HAVE_MM256_EXTRACT_EPI64) + + cmake_pop_check_state() +endif() # USE_CPU_EXTENSIONS + +macro(simd_add_definition_if target definition) + if(${definition}) + target_compile_definitions(${target} PRIVATE -D${definition}) + endif(${definition}) +endmacro(simd_add_definition_if) + +# Configure private preprocessor definitions for SIMD-related features +# Does not set any processor feature codegen flags +function(simd_add_definitions target) + simd_add_definition_if(${target} HAVE_AVX2_INTRINSICS) + simd_add_definition_if(${target} HAVE_MM256_EXTRACT_EPI64) +endfunction(simd_add_definitions) + +# Adds source files only if AVX2 is supported. These files will be built with +# avx2 intrinsics enabled. +# Usage: simd_add_source_avx2(target file1.c file2.c ...) +function(simd_add_source_avx2 target) + foreach(file ${ARGN}) + target_sources(${target} PRIVATE ${file}) + set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS "${AVX2_CFLAGS}") + endforeach() +endfunction(simd_add_source_avx2) diff --git a/contrib/aws-cmake/AwsThreadAffinity.cmake b/contrib/aws-cmake/AwsThreadAffinity.cmake new file mode 100644 index 00000000000..9e53481272c --- /dev/null +++ b/contrib/aws-cmake/AwsThreadAffinity.cmake @@ -0,0 +1,50 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +include(CheckSymbolExists) + +# Check if the platform supports setting thread affinity +# (important for hitting full NIC entitlement on NUMA architectures) +function(aws_set_thread_affinity_method target) + + # Non-POSIX, Android, and Apple platforms do not support thread affinity. + if (NOT UNIX OR ANDROID OR APPLE) + target_compile_definitions(${target} PRIVATE + -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE) + return() + endif() + + cmake_push_check_state() + list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE) + list(APPEND CMAKE_REQUIRED_LIBRARIES pthread) + + set(headers "pthread.h") + # BSDs put nonportable pthread declarations in a separate header. + if(CMAKE_SYSTEM_NAME MATCHES BSD) + set(headers "${headers};pthread_np.h") + endif() + + # Using pthread attrs is the preferred method, but is glibc-specific. + check_symbol_exists(pthread_attr_setaffinity_np "${headers}" USE_PTHREAD_ATTR_SETAFFINITY) + if (USE_PTHREAD_ATTR_SETAFFINITY) + target_compile_definitions(${target} PRIVATE + -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_PTHREAD_ATTR) + return() + endif() + + # This method is still nonportable, but is supported by musl and BSDs. + check_symbol_exists(pthread_setaffinity_np "${headers}" USE_PTHREAD_SETAFFINITY) + if (USE_PTHREAD_SETAFFINITY) + target_compile_definitions(${target} PRIVATE + -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_PTHREAD) + return() + endif() + + # If we got here, we expected thread affinity support but didn't find it. + # We still build with degraded NUMA performance, but show a warning. + message(WARNING "No supported method for setting thread affinity") + target_compile_definitions(${target} PRIVATE + -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE) + + cmake_pop_check_state() +endfunction() diff --git a/contrib/aws-cmake/AwsThreadName.cmake b/contrib/aws-cmake/AwsThreadName.cmake new file mode 100644 index 00000000000..a67416b4f83 --- /dev/null +++ b/contrib/aws-cmake/AwsThreadName.cmake @@ -0,0 +1,61 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +include(CheckSymbolExists) + +# Check how the platform supports setting thread name +function(aws_set_thread_name_method target) + + if (WINDOWS) + # On Windows we do a runtime check, instead of compile-time check + return() + elseif (APPLE) + # All Apple platforms we support have the same function, so no need for compile-time check. + return() + endif() + + cmake_push_check_state() + list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE) + list(APPEND CMAKE_REQUIRED_LIBRARIES pthread) + + # The start of the test program + set(c_source_start " + #define _GNU_SOURCE + #include + + #if defined(__FreeBSD__) || defined(__NETBSD__) + #include + #endif + + int main() { + pthread_t thread_id; + ") + + # The end of the test program + set(c_source_end "}") + + # pthread_setname_np() usually takes 2 args + check_c_source_compiles(" + ${c_source_start} + pthread_setname_np(thread_id, \"asdf\"); + ${c_source_end}" + PTHREAD_SETNAME_TAKES_2ARGS) + if (PTHREAD_SETNAME_TAKES_2ARGS) + target_compile_definitions(${target} PRIVATE -DAWS_PTHREAD_SETNAME_TAKES_2ARGS) + return() + endif() + + # But on NetBSD it takes 3! + check_c_source_compiles(" + ${c_source_start} + pthread_setname_np(thread_id, \"asdf\", NULL); + ${c_source_end} + " PTHREAD_SETNAME_TAKES_3ARGS) + if (PTHREAD_SETNAME_TAKES_3ARGS) + target_compile_definitions(${target} PRIVATE -DAWS_PTHREAD_SETNAME_TAKES_3ARGS) + return() + endif() + + # And on many older/weirder platforms it's just not supported + cmake_pop_check_state() +endfunction() diff --git a/contrib/aws-cmake/CMakeLists.txt b/contrib/aws-cmake/CMakeLists.txt new file mode 100644 index 00000000000..52533cd6483 --- /dev/null +++ b/contrib/aws-cmake/CMakeLists.txt @@ -0,0 +1,376 @@ +set(ENABLE_AWS_S3_DEFAULT OFF) + +if(ENABLE_LIBRARIES AND (OS_LINUX OR OS_DARWIN) AND TARGET OpenSSL::Crypto) + set(ENABLE_AWS_S3_DEFAULT ON) +endif() + +option(ENABLE_AWS_S3 "Enable AWS S3" ${ENABLE_AWS_S3_DEFAULT}) + +if(ENABLE_AWS_S3) + if(NOT TARGET OpenSSL::Crypto) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use AWS SDK without OpenSSL") + elseif(NOT (OS_LINUX OR OS_DARWIN)) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use AWS SDK with platform ${CMAKE_SYSTEM_NAME}") + endif() +endif() + +if(NOT ENABLE_AWS_S3) + message(STATUS "Not using AWS S3") + return() +endif() + + +# Utilities. +include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsFeatureTests.cmake") +include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsThreadAffinity.cmake") +include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsThreadName.cmake") +include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsSIMD.cmake") + + +# Gather sources and options. +set(AWS_SOURCES) +set(AWS_PUBLIC_INCLUDES) +set(AWS_PRIVATE_INCLUDES) +set(AWS_PUBLIC_COMPILE_DEFS) +set(AWS_PRIVATE_COMPILE_DEFS) +set(AWS_PRIVATE_LIBS) + +if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") + list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DDEBUG_BUILD") +endif() + +set(ENABLE_OPENSSL_ENCRYPTION ON) +if (ENABLE_OPENSSL_ENCRYPTION) + list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DENABLE_OPENSSL_ENCRYPTION") +endif() + +set(USE_S2N ON) +if (USE_S2N) + list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DUSE_S2N") +endif() + + +# Directories. +SET(AWS_SDK_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws") +SET(AWS_SDK_CORE_DIR "${AWS_SDK_DIR}/aws-cpp-sdk-core") +SET(AWS_SDK_S3_DIR "${AWS_SDK_DIR}/aws-cpp-sdk-s3") + +SET(AWS_AUTH_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-auth") +SET(AWS_CAL_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-cal") +SET(AWS_CHECKSUMS_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-checksums") +SET(AWS_COMMON_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-common") +SET(AWS_COMPRESSION_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-compression") +SET(AWS_CRT_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-crt-cpp") +SET(AWS_EVENT_STREAM_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-event-stream") +SET(AWS_HTTP_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-http") +SET(AWS_IO_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-io") +SET(AWS_MQTT_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-mqtt") +SET(AWS_S2N_TLS_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-s2n-tls") +SET(AWS_S3_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-s3") +SET(AWS_SDKUTILS_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-sdkutils") + + +# aws-cpp-sdk-core +file(GLOB AWS_SDK_CORE_SRC + "${AWS_SDK_CORE_DIR}/source/*.cpp" + "${AWS_SDK_CORE_DIR}/source/auth/*.cpp" + "${AWS_SDK_CORE_DIR}/source/auth/bearer-token-provider/*.cpp" + "${AWS_SDK_CORE_DIR}/source/auth/signer/*.cpp" + "${AWS_SDK_CORE_DIR}/source/auth/signer-provider/*.cpp" + "${AWS_SDK_CORE_DIR}/source/client/*.cpp" + "${AWS_SDK_CORE_DIR}/source/config/*.cpp" + "${AWS_SDK_CORE_DIR}/source/config/defaults/*.cpp" + "${AWS_SDK_CORE_DIR}/source/endpoint/*.cpp" + "${AWS_SDK_CORE_DIR}/source/endpoint/internal/*.cpp" + "${AWS_SDK_CORE_DIR}/source/external/cjson/*.cpp" + "${AWS_SDK_CORE_DIR}/source/external/tinyxml2/*.cpp" + "${AWS_SDK_CORE_DIR}/source/http/*.cpp" + "${AWS_SDK_CORE_DIR}/source/http/standard/*.cpp" + "${AWS_SDK_CORE_DIR}/source/internal/*.cpp" + "${AWS_SDK_CORE_DIR}/source/monitoring/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/base64/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/crypto/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/crypto/openssl/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/crypto/factory/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/event/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/json/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/logging/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/memory/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/memory/stl/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/stream/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/threading/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/xml/*.cpp" +) + +if(OS_LINUX OR OS_DARWIN) + file(GLOB AWS_SDK_CORE_NET_SRC "${AWS_SDK_CORE_DIR}/source/net/linux-shared/*.cpp") + file(GLOB AWS_SDK_CORE_PLATFORM_SRC "${AWS_SDK_CORE_DIR}/source/platform/linux-shared/*.cpp") +else() + file(GLOB AWS_SDK_CORE_NET_SRC "${AWS_SDK_CORE_DIR}/source/net/*.cpp") + set(AWS_SDK_CORE_PLATFORM_SRC) +endif() + +OPTION(USE_AWS_MEMORY_MANAGEMENT "Aws memory management" OFF) +configure_file("${AWS_SDK_CORE_DIR}/include/aws/core/SDKConfig.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/include/aws/core/SDKConfig.h" @ONLY) + +list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_MAJOR=1") +list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_MINOR=10") +list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_PATCH=36") + +list(APPEND AWS_SOURCES ${AWS_SDK_CORE_SRC} ${AWS_SDK_CORE_NET_SRC} ${AWS_SDK_CORE_PLATFORM_SRC}) + +list(APPEND AWS_PUBLIC_INCLUDES + "${AWS_SDK_CORE_DIR}/include/" + "${CMAKE_CURRENT_BINARY_DIR}/include" +) + + +# aws-cpp-sdk-s3 +file(GLOB AWS_SDK_S3_SRC + "${AWS_SDK_S3_DIR}/source/*.cpp" + "${AWS_SDK_S3_DIR}/source/model/*.cpp" +) + +list(APPEND AWS_SOURCES ${AWS_SDK_S3_SRC}) +list(APPEND AWS_PUBLIC_INCLUDES "${AWS_SDK_S3_DIR}/include/") + + +# aws-c-auth +file(GLOB AWS_AUTH_SRC + "${AWS_AUTH_DIR}/source/*.c" +) + +list(APPEND AWS_SOURCES ${AWS_AUTH_SRC}) +list(APPEND AWS_PUBLIC_INCLUDES "${AWS_AUTH_DIR}/include/") + + +# aws-c-cal +file(GLOB AWS_CAL_SRC + "${AWS_CAL_DIR}/source/*.c" +) + +if (ENABLE_OPENSSL_ENCRYPTION) + file(GLOB AWS_CAL_OS_SRC + "${AWS_CAL_DIR}/source/unix/*.c" + ) + list(APPEND AWS_PRIVATE_LIBS OpenSSL::Crypto) +endif() + +list(APPEND AWS_SOURCES ${AWS_CAL_SRC} ${AWS_CAL_OS_SRC}) +list(APPEND AWS_PRIVATE_INCLUDES "${AWS_CAL_DIR}/include/") + + +# aws-c-event-stream +file(GLOB AWS_EVENT_STREAM_SRC + "${AWS_EVENT_STREAM_DIR}/source/*.c" +) + +list(APPEND AWS_SOURCES ${AWS_EVENT_STREAM_SRC}) +list(APPEND AWS_PRIVATE_INCLUDES "${AWS_EVENT_STREAM_DIR}/include/") + + +# aws-c-common +file(GLOB AWS_COMMON_SRC + "${AWS_COMMON_DIR}/source/*.c" + "${AWS_COMMON_DIR}/source/external/*.c" + "${AWS_COMMON_DIR}/source/posix/*.c" +) + +file(GLOB AWS_COMMON_ARCH_SRC + "${AWS_COMMON_DIR}/source/arch/generic/*.c" +) + +if (AWS_ARCH_INTEL) + file(GLOB AWS_COMMON_ARCH_SRC + "${AWS_COMMON_DIR}/source/arch/intel/cpuid.c" + "${AWS_COMMON_DIR}/source/arch/intel/asm/*.c" + ) +elseif (AWS_ARCH_ARM64 OR AWS_ARCH_ARM32) + if (AWS_HAVE_AUXV) + file(GLOB AWS_COMMON_ARCH_SRC + "${AWS_COMMON_DIR}/source/arch/arm/asm/*.c" + ) + endif() +endif() + +set(AWS_COMMON_AVX2_SRC) +if (HAVE_AVX2_INTRINSICS) + list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DUSE_SIMD_ENCODING") + set(AWS_COMMON_AVX2_SRC "${AWS_COMMON_DIR}/source/arch/intel/encoding_avx2.c") + set_source_files_properties(${AWS_COMMON_AVX2_SRC} PROPERTIES COMPILE_FLAGS "${AVX2_CFLAGS}") +endif() + +configure_file("${AWS_COMMON_DIR}/include/aws/common/config.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/include/aws/common/config.h" @ONLY) + +list(APPEND AWS_SOURCES ${AWS_COMMON_SRC} ${AWS_COMMON_ARCH_SRC} ${AWS_COMMON_AVX2_SRC}) + +list(APPEND AWS_PUBLIC_INCLUDES + "${AWS_COMMON_DIR}/include/" + "${CMAKE_CURRENT_BINARY_DIR}/include" +) + + +# aws-checksums +file(GLOB AWS_CHECKSUMS_SRC + "${AWS_CHECKSUMS_DIR}/source/*.c" + "${AWS_CHECKSUMS_DIR}/source/intel/*.c" + "${AWS_CHECKSUMS_DIR}/source/intel/asm/*.c" + "${AWS_CHECKSUMS_DIR}/source/arm/*.c" +) + +if(AWS_ARCH_INTEL AND AWS_HAVE_GCC_INLINE_ASM) + file(GLOB AWS_CHECKSUMS_ARCH_SRC + "${AWS_CHECKSUMS_DIR}/source/intel/asm/*.c" + ) +endif() + +if (AWS_ARCH_ARM64) + file(GLOB AWS_CHECKSUMS_ARCH_SRC + "${AWS_CHECKSUMS_DIR}/source/arm/*.c" + ) + set_source_files_properties("${AWS_CHECKSUMS_DIR}/source/arm/crc32c_arm.c" PROPERTIES COMPILE_FLAGS -march=armv8-a+crc) +elseif (AWS_ARCH_ARM32) + if (AWS_ARM32_CRC) + file(GLOB AWS_CHECKSUMS_ARCH_SRC + "${AWS_CHECKSUMS_DIR}/source/arm/*.c" + "${AWS_CHECKSUMS_DIR}/source/arm/asm/*.c" + ) + set_source_files_properties(source/arm/crc32c_arm.c PROPERTIES COMPILE_FLAGS -march=armv8-a+crc) + endif() +endif() + +list(APPEND AWS_SOURCES ${AWS_CHECKSUMS_SRC} ${AWS_CHECKSUMS_ARCH_SRC}) +list(APPEND AWS_PRIVATE_INCLUDES "${AWS_CHECKSUMS_DIR}/include/") + + +# aws-c-io +file(GLOB AWS_IO_SRC + "${AWS_IO_DIR}/source/*.c" +) + +if (OS_LINUX) + file(GLOB AWS_IO_OS_SRC + "${AWS_IO_DIR}/source/linux/*.c" + "${AWS_IO_DIR}/source/posix/*.c" + ) +elseif (OS_DARWIN) + file(GLOB AWS_IO_OS_SRC + "${AWS_IO_DIR}/source/bsd/*.c" + "${AWS_IO_DIR}/source/posix/*.c" + ) +endif() + +set(AWS_IO_TLS_SRC) +if (USE_S2N) + file(GLOB AWS_IO_TLS_SRC + "${AWS_IO_DIR}/source/s2n/*.c" + ) +endif() + +list(APPEND AWS_SOURCES ${AWS_IO_SRC} ${AWS_IO_OS_SRC} ${AWS_IO_TLS_SRC}) +list(APPEND AWS_PUBLIC_INCLUDES "${AWS_IO_DIR}/include/") + + +# aws-s2n-tls +if (USE_S2N) + file(GLOB AWS_S2N_TLS_SRC + "${AWS_S2N_TLS_DIR}/crypto/*.c" + "${AWS_S2N_TLS_DIR}/error/*.c" + "${AWS_S2N_TLS_DIR}/stuffer/*.c" + "${AWS_S2N_TLS_DIR}/pq-crypto/*.c" + "${AWS_S2N_TLS_DIR}/pq-crypto/kyber_r3/*.c" + "${AWS_S2N_TLS_DIR}/tls/*.c" + "${AWS_S2N_TLS_DIR}/tls/extensions/*.c" + "${AWS_S2N_TLS_DIR}/utils/*.c" + ) + + list(APPEND AWS_SOURCES ${AWS_S2N_TLS_SRC}) + + list(APPEND AWS_PRIVATE_INCLUDES + "${AWS_S2N_TLS_DIR}/" + "${AWS_S2N_TLS_DIR}/api/" + ) +endif() + + +# aws-crt-cpp +file(GLOB AWS_CRT_SRC + "${AWS_CRT_DIR}/source/*.cpp" + "${AWS_CRT_DIR}/source/auth/*.cpp" + "${AWS_CRT_DIR}/source/crypto/*.cpp" + "${AWS_CRT_DIR}/source/endpoints/*.cpp" + "${AWS_CRT_DIR}/source/external/*.cpp" + "${AWS_CRT_DIR}/source/http/*.cpp" + "${AWS_CRT_DIR}/source/io/*.cpp" +) + +list(APPEND AWS_SOURCES ${AWS_CRT_SRC}) +list(APPEND AWS_PUBLIC_INCLUDES "${AWS_CRT_DIR}/include/") + + +# aws-c-mqtt +file(GLOB AWS_MQTT_SRC + "${AWS_MQTT_DIR}/source/*.c" +) + +list(APPEND AWS_SOURCES ${AWS_MQTT_SRC}) +list(APPEND AWS_PUBLIC_INCLUDES "${AWS_MQTT_DIR}/include/") + + +# aws-c-http +file(GLOB AWS_HTTP_SRC + "${AWS_HTTP_DIR}/source/*.c" +) + +list(APPEND AWS_SOURCES ${AWS_HTTP_SRC}) +list(APPEND AWS_PRIVATE_INCLUDES "${AWS_HTTP_DIR}/include/") + + +# aws-c-compression +file(GLOB AWS_COMPRESSION_SRC + "${AWS_COMPRESSION_DIR}/source/*.c" +) + +list(APPEND AWS_SOURCES ${AWS_COMPRESSION_SRC}) +list(APPEND AWS_PRIVATE_INCLUDES "${AWS_COMPRESSION_DIR}/include/") + + +# aws-c-s3 +file(GLOB AWS_S3_SRC + "${AWS_S3_DIR}/source/*.c" +) + +list(APPEND AWS_SOURCES ${AWS_S3_SRC}) +list(APPEND AWS_PRIVATE_INCLUDES "${AWS_S3_DIR}/include/") + + +# aws-c-sdkutils +file(GLOB AWS_SDKUTILS_SRC + "${AWS_SDKUTILS_DIR}/source/*.c" +) + +list(APPEND AWS_SOURCES ${AWS_SDKUTILS_SRC}) +list(APPEND AWS_PUBLIC_INCLUDES "${AWS_SDKUTILS_DIR}/include/") + + +# Add library. +add_library(_aws ${AWS_SOURCES}) + +target_include_directories(_aws SYSTEM BEFORE PUBLIC ${AWS_PUBLIC_INCLUDES}) +target_include_directories(_aws SYSTEM BEFORE PRIVATE ${AWS_PRIVATE_INCLUDES}) +target_compile_definitions(_aws PUBLIC ${AWS_PUBLIC_COMPILE_DEFS}) +target_compile_definitions(_aws PRIVATE ${AWS_PRIVATE_COMPILE_DEFS}) +target_link_libraries(_aws PRIVATE ${AWS_PRIVATE_LIBS}) + +aws_set_thread_affinity_method(_aws) +aws_set_thread_name_method(_aws) + +# The library is large - avoid bloat. +if (OMIT_HEAVY_DEBUG_SYMBOLS) + target_compile_options (_aws PRIVATE -g0) +endif() + +add_library(ch_contrib::aws_s3 ALIAS _aws) diff --git a/contrib/aws-crt-cpp b/contrib/aws-crt-cpp new file mode 160000 index 00000000000..ec0bea288f4 --- /dev/null +++ b/contrib/aws-crt-cpp @@ -0,0 +1 @@ +Subproject commit ec0bea288f451d884c0d80d534bc5c66241c39a4 diff --git a/contrib/aws-s2n-tls b/contrib/aws-s2n-tls new file mode 160000 index 00000000000..15d534e8a9c --- /dev/null +++ b/contrib/aws-s2n-tls @@ -0,0 +1 @@ +Subproject commit 15d534e8a9ca1eda6bacee514e37d08b4f38a526 diff --git a/contrib/aws-s3-cmake/CMakeLists.txt b/contrib/aws-s3-cmake/CMakeLists.txt deleted file mode 100644 index eabed601722..00000000000 --- a/contrib/aws-s3-cmake/CMakeLists.txt +++ /dev/null @@ -1,122 +0,0 @@ -if(NOT OS_FREEBSD) - option(ENABLE_S3 "Enable S3" ${ENABLE_LIBRARIES}) -elseif(ENABLE_S3) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use S3 on FreeBSD") -endif() - -if(NOT ENABLE_S3) - message(STATUS "Not using S3") - return() -endif() - -SET(AWS_S3_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3") -SET(AWS_CORE_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-core") -SET(AWS_CHECKSUMS_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-checksums") -SET(AWS_COMMON_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-common") -SET(AWS_EVENT_STREAM_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-event-stream") - -OPTION(USE_AWS_MEMORY_MANAGEMENT "Aws memory management" OFF) -configure_file("${AWS_CORE_LIBRARY_DIR}/include/aws/core/SDKConfig.h.in" - "${CMAKE_CURRENT_BINARY_DIR}/include/aws/core/SDKConfig.h" @ONLY) - -configure_file("${AWS_COMMON_LIBRARY_DIR}/include/aws/common/config.h.in" - "${CMAKE_CURRENT_BINARY_DIR}/include/aws/common/config.h" @ONLY) - - -file(GLOB AWS_CORE_SOURCES - "${AWS_CORE_LIBRARY_DIR}/source/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/auth/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/client/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/http/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/http/standard/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/config/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/external/cjson/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/external/tinyxml2/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/internal/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/monitoring/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/net/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/linux-shared/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/platform/linux-shared/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/utils/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/utils/base64/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/utils/event/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/openssl/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/factory/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/utils/json/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/utils/logging/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/utils/memory/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/utils/memory/stl/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/utils/stream/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/utils/threading/*.cpp" - "${AWS_CORE_LIBRARY_DIR}/source/utils/xml/*.cpp" -) - -file(GLOB AWS_S3_SOURCES - "${AWS_S3_LIBRARY_DIR}/source/*.cpp" -) - -file(GLOB AWS_S3_MODEL_SOURCES - "${AWS_S3_LIBRARY_DIR}/source/model/*.cpp" -) - -file(GLOB AWS_EVENT_STREAM_SOURCES - "${AWS_EVENT_STREAM_LIBRARY_DIR}/source/*.c" -) - -file(GLOB AWS_COMMON_SOURCES - "${AWS_COMMON_LIBRARY_DIR}/source/*.c" - "${AWS_COMMON_LIBRARY_DIR}/source/posix/*.c" -) - -file(GLOB AWS_CHECKSUMS_SOURCES - "${AWS_CHECKSUMS_LIBRARY_DIR}/source/*.c" - "${AWS_CHECKSUMS_LIBRARY_DIR}/source/intel/*.c" - "${AWS_CHECKSUMS_LIBRARY_DIR}/source/arm/*.c" -) - -file(GLOB S3_UNIFIED_SRC - ${AWS_EVENT_STREAM_SOURCES} - ${AWS_COMMON_SOURCES} - ${AWS_S3_SOURCES} - ${AWS_S3_MODEL_SOURCES} - ${AWS_CORE_SOURCES} -) - -set(S3_INCLUDES - "${AWS_COMMON_LIBRARY_DIR}/include/" - "${AWS_EVENT_STREAM_LIBRARY_DIR}/include/" - "${AWS_S3_LIBRARY_DIR}/include/" - "${AWS_CORE_LIBRARY_DIR}/include/" - "${CMAKE_CURRENT_BINARY_DIR}/include/" -) - -add_library(_aws_s3_checksums ${AWS_CHECKSUMS_SOURCES}) -target_include_directories(_aws_s3_checksums SYSTEM PUBLIC "${AWS_CHECKSUMS_LIBRARY_DIR}/include/") -if(CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") - target_compile_definitions(_aws_s3_checksums PRIVATE "-DDEBUG_BUILD") -endif() -set_target_properties(_aws_s3_checksums PROPERTIES LINKER_LANGUAGE C) -set_property(TARGET _aws_s3_checksums PROPERTY C_STANDARD 99) - -add_library(_aws_s3 ${S3_UNIFIED_SRC}) - -target_compile_definitions(_aws_s3 PUBLIC "AWS_SDK_VERSION_MAJOR=1") -target_compile_definitions(_aws_s3 PUBLIC "AWS_SDK_VERSION_MINOR=7") -target_compile_definitions(_aws_s3 PUBLIC "AWS_SDK_VERSION_PATCH=231") -target_include_directories(_aws_s3 SYSTEM BEFORE PUBLIC ${S3_INCLUDES}) - -if (TARGET OpenSSL::SSL) - target_compile_definitions(_aws_s3 PUBLIC -DENABLE_OPENSSL_ENCRYPTION) - target_link_libraries(_aws_s3 PRIVATE OpenSSL::Crypto OpenSSL::SSL) -endif() - -target_link_libraries(_aws_s3 PRIVATE _aws_s3_checksums) - -# The library is large - avoid bloat. -if (OMIT_HEAVY_DEBUG_SYMBOLS) - target_compile_options (_aws_s3 PRIVATE -g0) - target_compile_options (_aws_s3_checksums PRIVATE -g0) -endif() - -add_library(ch_contrib::aws_s3 ALIAS _aws_s3) diff --git a/contrib/sysroot b/contrib/sysroot index 0f41651860f..f0081b2649b 160000 --- a/contrib/sysroot +++ b/contrib/sysroot @@ -1 +1 @@ -Subproject commit 0f41651860fa4a530ecd68b93a15b8fd77397adf +Subproject commit f0081b2649b94837855f3bc7d05ef326b100bad8 diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 33dd3250c9f..6153842520b 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -177,7 +177,7 @@ namespace bool checkRequestCanReturn2xxAndErrorInBody(Aws::Http::HttpRequest & request) { auto query_params = request.GetQueryStringParameters(); - if (request.HasHeader("z-amz-copy-source")) + if (request.HasHeader("x-amz-copy-source")) { /// CopyObject https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html if (query_params.empty()) diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index c03f7f07310..a1a4267496f 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -149,13 +149,12 @@ class AWSEC2MetadataClient : public Aws::Internal::AWSHttpResourceClient static constexpr char EC2_IMDS_TOKEN_TTL_DEFAULT_VALUE[] = "21600"; static constexpr char EC2_IMDS_TOKEN_TTL_HEADER[] = "x-aws-ec2-metadata-token-ttl-seconds"; - static constexpr char EC2_DEFAULT_METADATA_ENDPOINT[] = "http://169.254.169.254"; - public: /// See EC2MetadataClient. - explicit AWSEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration) + explicit AWSEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration, const char * endpoint_) : Aws::Internal::AWSHttpResourceClient(client_configuration) + , endpoint(endpoint_) , logger(&Poco::Logger::get("AWSEC2InstanceProfileConfigLoader")) { } @@ -180,7 +179,7 @@ public: { std::lock_guard locker(token_mutex); - LOG_TRACE(logger, "Getting default credentials for EC2 instance."); + LOG_TRACE(logger, "Getting default credentials for ec2 instance from {}", endpoint); auto result = GetResourceWithAWSWebServiceResult(endpoint.c_str(), EC2_SECURITY_CREDENTIALS_RESOURCE, nullptr); credentials_string = result.GetPayload(); if (result.GetResponseCode() == Aws::Http::HttpResponseCode::UNAUTHORIZED) @@ -286,12 +285,50 @@ public: } private: - const Aws::String endpoint = EC2_DEFAULT_METADATA_ENDPOINT; + const Aws::String endpoint; mutable std::recursive_mutex token_mutex; mutable Aws::String token; Poco::Logger * logger; }; +std::shared_ptr InitEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration) +{ + Aws::String ec2_metadata_service_endpoint = Aws::Environment::GetEnv("AWS_EC2_METADATA_SERVICE_ENDPOINT"); + auto * logger = &Poco::Logger::get("AWSEC2InstanceProfileConfigLoader"); + if (ec2_metadata_service_endpoint.empty()) + { + Aws::String ec2_metadata_service_endpoint_mode = Aws::Environment::GetEnv("AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE"); + if (ec2_metadata_service_endpoint_mode.length() == 0) + { + ec2_metadata_service_endpoint = "http://169.254.169.254"; //default to IPv4 default endpoint + } + else + { + if (ec2_metadata_service_endpoint_mode.length() == 4) + { + if (Aws::Utils::StringUtils::CaselessCompare(ec2_metadata_service_endpoint_mode.c_str(), "ipv4")) + { + ec2_metadata_service_endpoint = "http://169.254.169.254"; //default to IPv4 default endpoint + } + else if (Aws::Utils::StringUtils::CaselessCompare(ec2_metadata_service_endpoint_mode.c_str(), "ipv6")) + { + ec2_metadata_service_endpoint = "http://[fd00:ec2::254]"; + } + else + { + LOG_ERROR(logger, "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE can only be set to ipv4 or ipv6, received: {}", ec2_metadata_service_endpoint_mode); + } + } + else + { + LOG_ERROR(logger, "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE can only be set to ipv4 or ipv6, received: {}", ec2_metadata_service_endpoint_mode); + } + } + } + LOG_INFO(logger, "Using IMDS endpoint: {}", ec2_metadata_service_endpoint); + return std::make_shared(client_configuration, ec2_metadata_service_endpoint.c_str()); +} + class AWSEC2InstanceProfileConfigLoader : public Aws::Config::AWSProfileConfigLoader { public: @@ -646,7 +683,7 @@ public: aws_client_configuration.retryStrategy = std::make_shared(1, 1000); - auto ec2_metadata_client = std::make_shared(aws_client_configuration); + auto ec2_metadata_client = InitEC2MetadataClient(aws_client_configuration); auto config_loader = std::make_shared(ec2_metadata_client, !use_insecure_imds_request); AddProvider(std::make_shared(config_loader)); diff --git a/tests/integration/test_s3_ec2_metadata/__init__.py b/tests/integration/test_s3_ec2_metadata/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_s3_ec2_metadata/configs/use_environment_credentials.xml b/tests/integration/test_s3_ec2_metadata/configs/use_environment_credentials.xml new file mode 100644 index 00000000000..db901ec0823 --- /dev/null +++ b/tests/integration/test_s3_ec2_metadata/configs/use_environment_credentials.xml @@ -0,0 +1,5 @@ + + + 1 + + diff --git a/tests/integration/test_s3_ec2_metadata/ec2_metadata_server/request_response_server.py b/tests/integration/test_s3_ec2_metadata/ec2_metadata_server/request_response_server.py new file mode 100644 index 00000000000..f347866be58 --- /dev/null +++ b/tests/integration/test_s3_ec2_metadata/ec2_metadata_server/request_response_server.py @@ -0,0 +1,36 @@ +import http.server +import sys + + +class RequestHandler(http.server.BaseHTTPRequestHandler): + def get_response(self): + if self.path == "/": + return "OK" + elif self.path == "/latest/meta-data/iam/security-credentials": + return "myrole" + elif self.path == "/latest/meta-data/iam/security-credentials/myrole": + return '{ "Code" : "Success", "Type" : "AWS-HMAC", "AccessKeyId" : "minio", "SecretAccessKey" : "minio123" }' + else: + return None + + def do_HEAD(self): + response = self.get_response() + if response: + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", len(response.encode())) + self.end_headers() + else: + self.send_response(404) + self.send_header("Content-Type", "text/plain") + self.end_headers() + + def do_GET(self): + self.do_HEAD() + response = self.get_response() + if response: + self.wfile.write(response.encode()) + + +httpd = http.server.HTTPServer(("0.0.0.0", int(sys.argv[1])), RequestHandler) +httpd.serve_forever() diff --git a/tests/integration/test_s3_ec2_metadata/test.py b/tests/integration/test_s3_ec2_metadata/test.py new file mode 100644 index 00000000000..982656df009 --- /dev/null +++ b/tests/integration/test_s3_ec2_metadata/test.py @@ -0,0 +1,94 @@ +import pytest +from helpers.cluster import ClickHouseCluster +import logging +import os +import time + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + +EC2_METADATA_SERVER_HOSTNAME = "resolver" +EC2_METADATA_SERVER_PORT = 8080 + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance( + "node", + with_minio=True, + main_configs=["configs/use_environment_credentials.xml"], + env_variables={ + "AWS_EC2_METADATA_SERVICE_ENDPOINT": f"{EC2_METADATA_SERVER_HOSTNAME}:{EC2_METADATA_SERVER_PORT}", + }, +) + + +def start_ec2_metadata_server(): + logging.info("Starting EC2 metadata server") + container_id = cluster.get_container_id("resolver") + + cluster.copy_file_to_container( + container_id, + os.path.join(SCRIPT_DIR, "ec2_metadata_server/request_response_server.py"), + "request_response_server.py", + ) + + cluster.exec_in_container( + container_id, + ["python", "request_response_server.py", str(EC2_METADATA_SERVER_PORT)], + detach=True, + ) + + # Wait for the server to start. + num_attempts = 100 + for attempt in range(num_attempts): + ping_response = cluster.exec_in_container( + container_id, + ["curl", "-s", f"http://localhost:{EC2_METADATA_SERVER_PORT}/"], + nothrow=True, + ) + if ping_response != "OK": + if attempt == num_attempts - 1: + assert ping_response == "OK", 'Expected "OK", but got "{}"'.format( + ping_response + ) + else: + time.sleep(1) + else: + logging.debug( + f"request_response_server.py answered {ping_response} on attempt {attempt}" + ) + break + + logging.info("EC2 metadata server started") + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + start_ec2_metadata_server() + yield + finally: + cluster.shutdown() + + +def test_credentials_from_ec2_metadata(): + node.query( + f"INSERT INTO FUNCTION s3('http://{cluster.minio_host}:{cluster.minio_port}/{cluster.minio_bucket}/test1.jsonl') SELECT * FROM numbers(100)" + ) + + assert ( + "100" + == node.query( + f"SELECT count() FROM s3('http://{cluster.minio_host}:{cluster.minio_port}/{cluster.minio_bucket}/test1.jsonl')" + ).strip() + ) + + expected_logs = [ + "Getting default credentials for ec2 instance from resolver:8080", + "Calling EC2MetadataService resource, /latest/meta-data/iam/security-credentials returned credential string myrole", + "Calling EC2MetadataService resource /latest/meta-data/iam/security-credentials/myrole", + "Successfully pulled credentials from EC2MetadataService with access key", + ] + + for expected_msg in expected_logs: + node.contains_in_log("AWSEC2InstanceProfileConfigLoader: " + expected_msg)