Merge pull request #43020 from vitlibar/update-aws-c-submodules

Update aws-c* submodules

.gitmodules

@@ -104,13 +104,13 @@
 	url = https://github.com/ClickHouse/aws-sdk-cpp.git
 [submodule "aws-c-event-stream"]
 	path = contrib/aws-c-event-stream
-	url = https://github.com/ClickHouse/aws-c-event-stream.git
+	url = https://github.com/awslabs/aws-c-event-stream.git
 [submodule "aws-c-common"]
 	path = contrib/aws-c-common
 	url = https://github.com/ClickHouse/aws-c-common.git
 [submodule "aws-checksums"]
 	path = contrib/aws-checksums
-	url = https://github.com/ClickHouse/aws-checksums.git
+	url = https://github.com/awslabs/aws-checksums.git
 [submodule "contrib/curl"]
 	path = contrib/curl
 	url = https://github.com/curl/curl.git
@@ -294,3 +294,33 @@
 [submodule "contrib/libdivide"]
 	path = contrib/libdivide
 	url = https://github.com/ridiculousfish/libdivide.git
+[submodule "contrib/aws-crt-cpp"]
+	path = contrib/aws-crt-cpp
+	url = https://github.com/ClickHouse/aws-crt-cpp.git
+[submodule "contrib/aws-c-io"]
+	path = contrib/aws-c-io
+	url = https://github.com/ClickHouse/aws-c-io.git
+[submodule "contrib/aws-c-mqtt"]
+	path = contrib/aws-c-mqtt
+	url = https://github.com/awslabs/aws-c-mqtt.git
+[submodule "contrib/aws-c-auth"]
+	path = contrib/aws-c-auth
+	url = https://github.com/awslabs/aws-c-auth.git
+[submodule "contrib/aws-c-cal"]
+	path = contrib/aws-c-cal
+	url = https://github.com/ClickHouse/aws-c-cal.git
+[submodule "contrib/aws-c-sdkutils"]
+	path = contrib/aws-c-sdkutils
+	url = https://github.com/awslabs/aws-c-sdkutils.git
+[submodule "contrib/aws-c-http"]
+	path = contrib/aws-c-http
+	url = https://github.com/awslabs/aws-c-http.git
+[submodule "contrib/aws-c-s3"]
+	path = contrib/aws-c-s3
+	url = https://github.com/awslabs/aws-c-s3.git
+[submodule "contrib/aws-c-compression"]
+	path = contrib/aws-c-compression
+	url = https://github.com/awslabs/aws-c-compression.git
+[submodule "contrib/aws-s2n-tls"]
+	path = contrib/aws-s2n-tls
+	url = https://github.com/aws/s2n-tls.git

contrib/CMakeLists.txt

@@ -115,12 +115,25 @@ endif()
 add_contrib (llvm-project-cmake llvm-project)
 add_contrib (libfuzzer-cmake llvm-project)
 add_contrib (libxml2-cmake libxml2)
-add_contrib (aws-s3-cmake
+
+add_contrib (aws-cmake
     aws
+    aws-c-auth
+    aws-c-cal
     aws-c-common
+    aws-c-compression
     aws-c-event-stream
+    aws-c-http
+    aws-c-io
+    aws-c-mqtt
+    aws-c-s3
+    aws-c-sdkutils
+    aws-s2n-tls
     aws-checksums
+    aws-crt-cpp
+    aws-cmake
 )
+
 add_contrib (base64-cmake base64)
 add_contrib (simdjson-cmake simdjson)
 add_contrib (rapidjson-cmake rapidjson)

contrib/aws

@@ -1 +1 @@
-Subproject commit 00b03604543367d7e310cb0993973fdcb723ea79
+Subproject commit 4a12641211d4dbc8e2fdb2dd0f1eea0927db9252

contrib/aws-c-auth

@@ -0,0 +1 @@
+Subproject commit 30df6c407e2df43bd244e2c34c9b4a4b87372bfb

contrib/aws-c-cal

@@ -0,0 +1 @@
+Subproject commit 85dd7664b786a389c6fb1a6f031ab4bb2282133d

contrib/aws-c-common

@@ -1 +1 @@
-Subproject commit 736a82d1697c108b04a277e66438a7f4e19b6857
+Subproject commit 324fd1d973ccb25c813aa747bf1759cfde5121c5

contrib/aws-c-compression

@@ -0,0 +1 @@
+Subproject commit b517b7decd0dac30be2162f5186c250221c53aff

contrib/aws-c-event-stream

@@ -1 +1 @@
-Subproject commit 3bc33662f9ccff4f4cbcf9509cc78c26e022fde0
+Subproject commit 39bfa94a14b7126bf0c1330286ef8db452d87e66

contrib/aws-c-http

@@ -0,0 +1 @@
+Subproject commit 2c5a2a7d5556600b9782ffa6c9d7e09964df1abc

contrib/aws-c-io

@@ -0,0 +1 @@
+Subproject commit 5d32c453560d0823df521a686bf7fbacde7f9be3

contrib/aws-c-mqtt

@@ -0,0 +1 @@
+Subproject commit 882c689561a3db1466330ccfe3b63637e0a575d3

contrib/aws-c-s3

@@ -0,0 +1 @@
+Subproject commit a41255ece72a7c887bba7f9d998ca3e14f4c8a1b

contrib/aws-c-sdkutils

@@ -0,0 +1 @@
+Subproject commit 25bf5cf225f977c3accc6a05a0a7a181ef2a4a30

contrib/aws-checksums

@@ -1 +1 @@
-Subproject commit 519d6d9093819b6cf89ffff589a27ef8f83d0f65
+Subproject commit 48e7c0e01479232f225c8044d76c84e74192889d

contrib/aws-cmake/AwsFeatureTests.cmake

@@ -0,0 +1,114 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+include(CheckCSourceRuns)
+
+option(USE_CPU_EXTENSIONS "Whenever possible, use functions optimized for CPUs with specific extensions (ex: SSE, AVX)." ON)
+
+# In the current (11/2/21) state of mingw64, the packaged gcc is not capable of emitting properly aligned avx2 instructions under certain circumstances.
+# This leads to crashes for windows builds using mingw64 when invoking the avx2-enabled versions of certain functions.  Until we can find a better
+# work-around, disable avx2 (and all other extensions) in mingw builds.
+#
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412
+#
+if (MINGW)
+    message(STATUS "MINGW detected!  Disabling avx2 and other CPU extensions")
+    set(USE_CPU_EXTENSIONS OFF)
+endif()
+
+if(NOT CMAKE_CROSSCOMPILING)
+    check_c_source_runs("
+    #include <stdbool.h>
+    bool foo(int a, int b, int *c) {
+        return __builtin_mul_overflow(a, b, c);
+    }
+
+    int main() {
+        int out;
+        if (foo(1, 2, &out)) {
+            return 0;
+        }
+
+        return 0;
+    }" AWS_HAVE_GCC_OVERFLOW_MATH_EXTENSIONS)
+
+    if (USE_CPU_EXTENSIONS)
+        check_c_source_runs("
+        int main() {
+        int foo = 42;
+        _mulx_u32(1, 2, &foo);
+        return foo != 2;
+        }" AWS_HAVE_MSVC_MULX)
+    endif()
+
+endif()
+
+check_c_source_compiles("
+    #include <Windows.h>
+    #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
+    int main() {
+        return 0;
+    }
+    #else
+    it's not windows desktop
+    #endif
+" AWS_HAVE_WINAPI_DESKTOP)
+
+check_c_source_compiles("
+    int main() {
+#if !(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
+#    error \"not intel\"
+#endif
+        return 0;
+    }
+" AWS_ARCH_INTEL)
+
+check_c_source_compiles("
+    int main() {
+#if !(defined(__aarch64__) || defined(_M_ARM64))
+#    error \"not arm64\"
+#endif
+        return 0;
+    }
+" AWS_ARCH_ARM64)
+
+check_c_source_compiles("
+    int main() {
+#if !(defined(__arm__) || defined(_M_ARM))
+#    error \"not arm\"
+#endif
+        return 0;
+    }
+" AWS_ARCH_ARM32)
+
+check_c_source_compiles("
+int main() {
+    int foo = 42, bar = 24;
+    __asm__ __volatile__(\"\":\"=r\"(foo):\"r\"(bar):\"memory\");
+}" AWS_HAVE_GCC_INLINE_ASM)
+
+check_c_source_compiles("
+#include <sys/auxv.h>
+int main() {
+#ifdef __linux__
+    getauxval(AT_HWCAP);
+    getauxval(AT_HWCAP2);
+#endif
+    return 0;
+}" AWS_HAVE_AUXV)
+
+string(REGEX MATCH "^(aarch64|arm)" ARM_CPU "${CMAKE_SYSTEM_PROCESSOR}")
+if(NOT LEGACY_COMPILER_SUPPORT OR ARM_CPU)
+    check_c_source_compiles("
+    #include <execinfo.h>
+    int main() {
+        backtrace(NULL, 0);
+        return 0;
+    }" AWS_HAVE_EXECINFO)
+endif()
+
+check_c_source_compiles("
+#include <linux/if_link.h>
+int main() {
+    return 1;
+}" AWS_HAVE_LINUX_IF_LINK_H)

contrib/aws-cmake/AwsSIMD.cmake

@@ -0,0 +1,74 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+include(CheckCCompilerFlag)
+include(CheckIncludeFile)
+
+if (USE_CPU_EXTENSIONS)
+    if (MSVC)
+        check_c_compiler_flag("/arch:AVX2" HAVE_M_AVX2_FLAG)
+        if (HAVE_M_AVX2_FLAG)
+            set(AVX2_CFLAGS "/arch:AVX2")
+        endif()
+    else()
+        check_c_compiler_flag(-mavx2 HAVE_M_AVX2_FLAG)
+        if (HAVE_M_AVX2_FLAG)
+            set(AVX2_CFLAGS "-mavx -mavx2")
+        endif()
+    endif()
+
+
+    cmake_push_check_state()
+    set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${AVX2_CFLAGS}")
+
+    check_c_source_compiles("
+        #include <immintrin.h>
+        #include <emmintrin.h>
+        #include <string.h>
+
+        int main() {
+            __m256i vec;
+            memset(&vec, 0, sizeof(vec));
+
+            _mm256_shuffle_epi8(vec, vec);
+            _mm256_set_epi32(1,2,3,4,5,6,7,8);
+            _mm256_permutevar8x32_epi32(vec, vec);
+
+            return 0;
+        }"  HAVE_AVX2_INTRINSICS)
+
+    check_c_source_compiles("
+        #include <immintrin.h>
+        #include <string.h>
+
+        int main() {
+            __m256i vec;
+            memset(&vec, 0, sizeof(vec));
+            return (int)_mm256_extract_epi64(vec, 2);
+        }" HAVE_MM256_EXTRACT_EPI64)
+
+    cmake_pop_check_state()
+endif() # USE_CPU_EXTENSIONS
+
+macro(simd_add_definition_if target definition)
+    if(${definition})
+        target_compile_definitions(${target} PRIVATE -D${definition})
+    endif(${definition})
+endmacro(simd_add_definition_if)
+
+# Configure private preprocessor definitions for SIMD-related features
+# Does not set any processor feature codegen flags
+function(simd_add_definitions target)
+    simd_add_definition_if(${target} HAVE_AVX2_INTRINSICS)
+    simd_add_definition_if(${target} HAVE_MM256_EXTRACT_EPI64)
+endfunction(simd_add_definitions)
+
+# Adds source files only if AVX2 is supported. These files will be built with
+# avx2 intrinsics enabled.
+# Usage: simd_add_source_avx2(target file1.c file2.c ...)
+function(simd_add_source_avx2 target)
+    foreach(file ${ARGN})
+        target_sources(${target} PRIVATE ${file})
+        set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS "${AVX2_CFLAGS}")
+    endforeach()
+endfunction(simd_add_source_avx2)

contrib/aws-cmake/AwsThreadAffinity.cmake

@@ -0,0 +1,50 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+include(CheckSymbolExists)
+
+# Check if the platform supports setting thread affinity
+# (important for hitting full NIC entitlement on NUMA architectures)
+function(aws_set_thread_affinity_method target)
+
+    # Non-POSIX, Android, and Apple platforms do not support thread affinity.
+    if (NOT UNIX OR ANDROID OR APPLE)
+        target_compile_definitions(${target} PRIVATE
+            -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE)
+        return()
+    endif()
+
+    cmake_push_check_state()
+    list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE)
+    list(APPEND CMAKE_REQUIRED_LIBRARIES pthread)
+
+    set(headers "pthread.h")
+    # BSDs put nonportable pthread declarations in a separate header.
+    if(CMAKE_SYSTEM_NAME MATCHES BSD)
+        set(headers "${headers};pthread_np.h")
+    endif()
+
+    # Using pthread attrs is the preferred method, but is glibc-specific.
+    check_symbol_exists(pthread_attr_setaffinity_np "${headers}" USE_PTHREAD_ATTR_SETAFFINITY)
+    if (USE_PTHREAD_ATTR_SETAFFINITY)
+        target_compile_definitions(${target} PRIVATE
+            -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_PTHREAD_ATTR)
+        return()
+    endif()
+
+    # This method is still nonportable, but is supported by musl and BSDs.
+    check_symbol_exists(pthread_setaffinity_np "${headers}" USE_PTHREAD_SETAFFINITY)
+    if (USE_PTHREAD_SETAFFINITY)
+        target_compile_definitions(${target} PRIVATE
+            -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_PTHREAD)
+        return()
+    endif()
+
+    # If we got here, we expected thread affinity support but didn't find it.
+    # We still build with degraded NUMA performance, but show a warning.
+    message(WARNING "No supported method for setting thread affinity")
+    target_compile_definitions(${target} PRIVATE
+        -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE)
+
+    cmake_pop_check_state()
+endfunction()

contrib/aws-cmake/AwsThreadName.cmake

@@ -0,0 +1,61 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+include(CheckSymbolExists)
+
+# Check how the platform supports setting thread name
+function(aws_set_thread_name_method target)
+
+    if (WINDOWS)
+        # On Windows we do a runtime check, instead of compile-time check
+        return()
+    elseif (APPLE)
+        # All Apple platforms we support have the same function, so no need for compile-time check.
+        return()
+    endif()
+
+    cmake_push_check_state()
+    list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE)
+    list(APPEND CMAKE_REQUIRED_LIBRARIES pthread)
+
+    # The start of the test program
+    set(c_source_start "
+        #define _GNU_SOURCE
+        #include <pthread.h>
+
+        #if defined(__FreeBSD__) || defined(__NETBSD__)
+        #include <pthread_np.h>
+        #endif
+
+        int main() {
+            pthread_t thread_id;
+        ")
+
+    # The end of the test program
+    set(c_source_end "}")
+
+    # pthread_setname_np() usually takes 2 args
+    check_c_source_compiles("
+        ${c_source_start}
+        pthread_setname_np(thread_id, \"asdf\");
+        ${c_source_end}"
+        PTHREAD_SETNAME_TAKES_2ARGS)
+    if (PTHREAD_SETNAME_TAKES_2ARGS)
+        target_compile_definitions(${target} PRIVATE -DAWS_PTHREAD_SETNAME_TAKES_2ARGS)
+        return()
+    endif()
+
+    # But on NetBSD it takes 3!
+    check_c_source_compiles("
+        ${c_source_start}
+        pthread_setname_np(thread_id, \"asdf\", NULL);
+        ${c_source_end}
+        " PTHREAD_SETNAME_TAKES_3ARGS)
+    if (PTHREAD_SETNAME_TAKES_3ARGS)
+        target_compile_definitions(${target} PRIVATE -DAWS_PTHREAD_SETNAME_TAKES_3ARGS)
+        return()
+    endif()
+
+    # And on many older/weirder platforms it's just not supported
+    cmake_pop_check_state()
+endfunction()

contrib/aws-cmake/CMakeLists.txt

@@ -0,0 +1,376 @@
+set(ENABLE_AWS_S3_DEFAULT OFF)
+
+if(ENABLE_LIBRARIES AND (OS_LINUX OR OS_DARWIN) AND TARGET OpenSSL::Crypto)
+    set(ENABLE_AWS_S3_DEFAULT ON)
+endif()
+
+option(ENABLE_AWS_S3 "Enable AWS S3" ${ENABLE_AWS_S3_DEFAULT})
+
+if(ENABLE_AWS_S3)
+    if(NOT TARGET OpenSSL::Crypto)
+        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use AWS SDK without OpenSSL")
+    elseif(NOT (OS_LINUX OR OS_DARWIN))
+        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use AWS SDK with platform ${CMAKE_SYSTEM_NAME}")
+    endif()
+endif()
+
+if(NOT ENABLE_AWS_S3)
+    message(STATUS "Not using AWS S3")
+    return()
+endif()
+
+
+# Utilities.
+include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsFeatureTests.cmake")
+include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsThreadAffinity.cmake")
+include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsThreadName.cmake")
+include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsSIMD.cmake")
+
+
+# Gather sources and options.
+set(AWS_SOURCES)
+set(AWS_PUBLIC_INCLUDES)
+set(AWS_PRIVATE_INCLUDES)
+set(AWS_PUBLIC_COMPILE_DEFS)
+set(AWS_PRIVATE_COMPILE_DEFS)
+set(AWS_PRIVATE_LIBS)
+
+if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
+    list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DDEBUG_BUILD")
+endif()
+
+set(ENABLE_OPENSSL_ENCRYPTION ON)
+if (ENABLE_OPENSSL_ENCRYPTION)
+    list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DENABLE_OPENSSL_ENCRYPTION")
+endif()
+
+set(USE_S2N ON)
+if (USE_S2N)
+    list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DUSE_S2N")
+endif()
+
+
+# Directories.
+SET(AWS_SDK_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws")
+SET(AWS_SDK_CORE_DIR "${AWS_SDK_DIR}/aws-cpp-sdk-core")
+SET(AWS_SDK_S3_DIR "${AWS_SDK_DIR}/aws-cpp-sdk-s3")
+
+SET(AWS_AUTH_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-auth")
+SET(AWS_CAL_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-cal")
+SET(AWS_CHECKSUMS_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-checksums")
+SET(AWS_COMMON_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-common")
+SET(AWS_COMPRESSION_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-compression")
+SET(AWS_CRT_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-crt-cpp")
+SET(AWS_EVENT_STREAM_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-event-stream")
+SET(AWS_HTTP_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-http")
+SET(AWS_IO_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-io")
+SET(AWS_MQTT_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-mqtt")
+SET(AWS_S2N_TLS_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-s2n-tls")
+SET(AWS_S3_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-s3")
+SET(AWS_SDKUTILS_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-sdkutils")
+
+
+# aws-cpp-sdk-core
+file(GLOB AWS_SDK_CORE_SRC
+    "${AWS_SDK_CORE_DIR}/source/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/auth/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/auth/bearer-token-provider/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/auth/signer/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/auth/signer-provider/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/client/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/config/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/config/defaults/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/endpoint/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/endpoint/internal/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/external/cjson/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/external/tinyxml2/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/http/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/http/standard/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/internal/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/monitoring/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/utils/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/utils/base64/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/utils/crypto/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/utils/crypto/openssl/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/utils/crypto/factory/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/utils/event/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/utils/json/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/utils/logging/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/utils/memory/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/utils/memory/stl/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/utils/stream/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/utils/threading/*.cpp"
+    "${AWS_SDK_CORE_DIR}/source/utils/xml/*.cpp"
+)
+
+if(OS_LINUX OR OS_DARWIN)
+    file(GLOB AWS_SDK_CORE_NET_SRC "${AWS_SDK_CORE_DIR}/source/net/linux-shared/*.cpp")
+    file(GLOB AWS_SDK_CORE_PLATFORM_SRC "${AWS_SDK_CORE_DIR}/source/platform/linux-shared/*.cpp")
+else()
+    file(GLOB AWS_SDK_CORE_NET_SRC "${AWS_SDK_CORE_DIR}/source/net/*.cpp")
+    set(AWS_SDK_CORE_PLATFORM_SRC)
+endif()
+
+OPTION(USE_AWS_MEMORY_MANAGEMENT "Aws memory management" OFF)
+configure_file("${AWS_SDK_CORE_DIR}/include/aws/core/SDKConfig.h.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/include/aws/core/SDKConfig.h" @ONLY)
+
+list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_MAJOR=1")
+list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_MINOR=10")
+list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_PATCH=36")
+    
+list(APPEND AWS_SOURCES ${AWS_SDK_CORE_SRC} ${AWS_SDK_CORE_NET_SRC} ${AWS_SDK_CORE_PLATFORM_SRC})
+
+list(APPEND AWS_PUBLIC_INCLUDES
+    "${AWS_SDK_CORE_DIR}/include/"
+    "${CMAKE_CURRENT_BINARY_DIR}/include"
+)
+
+
+# aws-cpp-sdk-s3
+file(GLOB AWS_SDK_S3_SRC
+    "${AWS_SDK_S3_DIR}/source/*.cpp"
+    "${AWS_SDK_S3_DIR}/source/model/*.cpp"
+)
+
+list(APPEND AWS_SOURCES ${AWS_SDK_S3_SRC})
+list(APPEND AWS_PUBLIC_INCLUDES "${AWS_SDK_S3_DIR}/include/")
+
+
+# aws-c-auth
+file(GLOB AWS_AUTH_SRC
+    "${AWS_AUTH_DIR}/source/*.c"
+)
+
+list(APPEND AWS_SOURCES ${AWS_AUTH_SRC})
+list(APPEND AWS_PUBLIC_INCLUDES "${AWS_AUTH_DIR}/include/")
+
+
+# aws-c-cal
+file(GLOB AWS_CAL_SRC
+    "${AWS_CAL_DIR}/source/*.c"
+)
+
+if (ENABLE_OPENSSL_ENCRYPTION)
+    file(GLOB AWS_CAL_OS_SRC
+        "${AWS_CAL_DIR}/source/unix/*.c"
+    )
+    list(APPEND AWS_PRIVATE_LIBS OpenSSL::Crypto)
+endif()
+
+list(APPEND AWS_SOURCES ${AWS_CAL_SRC} ${AWS_CAL_OS_SRC})
+list(APPEND AWS_PRIVATE_INCLUDES "${AWS_CAL_DIR}/include/")
+
+
+# aws-c-event-stream
+file(GLOB AWS_EVENT_STREAM_SRC
+    "${AWS_EVENT_STREAM_DIR}/source/*.c"
+)
+
+list(APPEND AWS_SOURCES ${AWS_EVENT_STREAM_SRC})
+list(APPEND AWS_PRIVATE_INCLUDES "${AWS_EVENT_STREAM_DIR}/include/")
+
+
+# aws-c-common
+file(GLOB AWS_COMMON_SRC
+    "${AWS_COMMON_DIR}/source/*.c"
+    "${AWS_COMMON_DIR}/source/external/*.c"
+    "${AWS_COMMON_DIR}/source/posix/*.c"
+)
+
+file(GLOB AWS_COMMON_ARCH_SRC
+    "${AWS_COMMON_DIR}/source/arch/generic/*.c"
+)
+
+if (AWS_ARCH_INTEL)
+    file(GLOB AWS_COMMON_ARCH_SRC
+        "${AWS_COMMON_DIR}/source/arch/intel/cpuid.c"
+        "${AWS_COMMON_DIR}/source/arch/intel/asm/*.c"
+    )
+elseif (AWS_ARCH_ARM64 OR AWS_ARCH_ARM32)
+    if (AWS_HAVE_AUXV)
+        file(GLOB AWS_COMMON_ARCH_SRC
+            "${AWS_COMMON_DIR}/source/arch/arm/asm/*.c"
+        )
+    endif()
+endif()
+
+set(AWS_COMMON_AVX2_SRC)
+if (HAVE_AVX2_INTRINSICS)
+    list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DUSE_SIMD_ENCODING")
+    set(AWS_COMMON_AVX2_SRC "${AWS_COMMON_DIR}/source/arch/intel/encoding_avx2.c")
+    set_source_files_properties(${AWS_COMMON_AVX2_SRC} PROPERTIES COMPILE_FLAGS "${AVX2_CFLAGS}")
+endif()
+
+configure_file("${AWS_COMMON_DIR}/include/aws/common/config.h.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/include/aws/common/config.h" @ONLY)
+
+list(APPEND AWS_SOURCES ${AWS_COMMON_SRC} ${AWS_COMMON_ARCH_SRC} ${AWS_COMMON_AVX2_SRC})
+
+list(APPEND AWS_PUBLIC_INCLUDES
+    "${AWS_COMMON_DIR}/include/"
+    "${CMAKE_CURRENT_BINARY_DIR}/include"
+)
+
+
+# aws-checksums
+file(GLOB AWS_CHECKSUMS_SRC
+    "${AWS_CHECKSUMS_DIR}/source/*.c"
+    "${AWS_CHECKSUMS_DIR}/source/intel/*.c"
+    "${AWS_CHECKSUMS_DIR}/source/intel/asm/*.c"
+    "${AWS_CHECKSUMS_DIR}/source/arm/*.c"
+)
+
+if(AWS_ARCH_INTEL AND AWS_HAVE_GCC_INLINE_ASM)
+    file(GLOB AWS_CHECKSUMS_ARCH_SRC
+            "${AWS_CHECKSUMS_DIR}/source/intel/asm/*.c"
+    )
+endif()
+
+if (AWS_ARCH_ARM64)
+    file(GLOB AWS_CHECKSUMS_ARCH_SRC
+        "${AWS_CHECKSUMS_DIR}/source/arm/*.c"
+    )
+    set_source_files_properties("${AWS_CHECKSUMS_DIR}/source/arm/crc32c_arm.c" PROPERTIES COMPILE_FLAGS -march=armv8-a+crc)
+elseif (AWS_ARCH_ARM32)
+    if (AWS_ARM32_CRC)
+        file(GLOB AWS_CHECKSUMS_ARCH_SRC
+            "${AWS_CHECKSUMS_DIR}/source/arm/*.c"
+            "${AWS_CHECKSUMS_DIR}/source/arm/asm/*.c"
+        )
+        set_source_files_properties(source/arm/crc32c_arm.c PROPERTIES COMPILE_FLAGS -march=armv8-a+crc)
+    endif()
+endif()
+
+list(APPEND AWS_SOURCES ${AWS_CHECKSUMS_SRC} ${AWS_CHECKSUMS_ARCH_SRC})
+list(APPEND AWS_PRIVATE_INCLUDES "${AWS_CHECKSUMS_DIR}/include/")
+
+
+# aws-c-io
+file(GLOB AWS_IO_SRC
+    "${AWS_IO_DIR}/source/*.c"
+)
+
+if (OS_LINUX)
+    file(GLOB AWS_IO_OS_SRC
+        "${AWS_IO_DIR}/source/linux/*.c"
+        "${AWS_IO_DIR}/source/posix/*.c"
+    )
+elseif (OS_DARWIN)
+    file(GLOB AWS_IO_OS_SRC
+        "${AWS_IO_DIR}/source/bsd/*.c"
+        "${AWS_IO_DIR}/source/posix/*.c"
+    )
+endif()
+
+set(AWS_IO_TLS_SRC)
+if (USE_S2N)
+    file(GLOB AWS_IO_TLS_SRC
+            "${AWS_IO_DIR}/source/s2n/*.c"
+    )
+endif()
+
+list(APPEND AWS_SOURCES ${AWS_IO_SRC} ${AWS_IO_OS_SRC} ${AWS_IO_TLS_SRC})
+list(APPEND AWS_PUBLIC_INCLUDES "${AWS_IO_DIR}/include/")
+
+
+# aws-s2n-tls
+if (USE_S2N)
+    file(GLOB AWS_S2N_TLS_SRC
+        "${AWS_S2N_TLS_DIR}/crypto/*.c"
+        "${AWS_S2N_TLS_DIR}/error/*.c"
+        "${AWS_S2N_TLS_DIR}/stuffer/*.c"
+        "${AWS_S2N_TLS_DIR}/pq-crypto/*.c"
+        "${AWS_S2N_TLS_DIR}/pq-crypto/kyber_r3/*.c"
+        "${AWS_S2N_TLS_DIR}/tls/*.c"
+        "${AWS_S2N_TLS_DIR}/tls/extensions/*.c"
+        "${AWS_S2N_TLS_DIR}/utils/*.c"
+    )
+
+    list(APPEND AWS_SOURCES ${AWS_S2N_TLS_SRC})
+
+    list(APPEND AWS_PRIVATE_INCLUDES
+        "${AWS_S2N_TLS_DIR}/"
+        "${AWS_S2N_TLS_DIR}/api/"
+    )
+endif()
+
+
+# aws-crt-cpp
+file(GLOB AWS_CRT_SRC
+    "${AWS_CRT_DIR}/source/*.cpp"
+    "${AWS_CRT_DIR}/source/auth/*.cpp"
+    "${AWS_CRT_DIR}/source/crypto/*.cpp"
+    "${AWS_CRT_DIR}/source/endpoints/*.cpp"
+    "${AWS_CRT_DIR}/source/external/*.cpp"
+    "${AWS_CRT_DIR}/source/http/*.cpp"
+    "${AWS_CRT_DIR}/source/io/*.cpp"
+)
+
+list(APPEND AWS_SOURCES ${AWS_CRT_SRC})
+list(APPEND AWS_PUBLIC_INCLUDES "${AWS_CRT_DIR}/include/")
+
+
+# aws-c-mqtt
+file(GLOB AWS_MQTT_SRC
+    "${AWS_MQTT_DIR}/source/*.c"
+)
+
+list(APPEND AWS_SOURCES ${AWS_MQTT_SRC})
+list(APPEND AWS_PUBLIC_INCLUDES "${AWS_MQTT_DIR}/include/")
+
+
+# aws-c-http
+file(GLOB AWS_HTTP_SRC
+    "${AWS_HTTP_DIR}/source/*.c"
+)
+
+list(APPEND AWS_SOURCES ${AWS_HTTP_SRC})
+list(APPEND AWS_PRIVATE_INCLUDES "${AWS_HTTP_DIR}/include/")
+
+
+# aws-c-compression
+file(GLOB AWS_COMPRESSION_SRC
+    "${AWS_COMPRESSION_DIR}/source/*.c"
+)
+
+list(APPEND AWS_SOURCES ${AWS_COMPRESSION_SRC})
+list(APPEND AWS_PRIVATE_INCLUDES "${AWS_COMPRESSION_DIR}/include/")
+
+
+# aws-c-s3
+file(GLOB AWS_S3_SRC
+    "${AWS_S3_DIR}/source/*.c"
+)
+
+list(APPEND AWS_SOURCES ${AWS_S3_SRC})
+list(APPEND AWS_PRIVATE_INCLUDES "${AWS_S3_DIR}/include/")
+
+
+# aws-c-sdkutils
+file(GLOB AWS_SDKUTILS_SRC
+    "${AWS_SDKUTILS_DIR}/source/*.c"
+)
+
+list(APPEND AWS_SOURCES ${AWS_SDKUTILS_SRC})
+list(APPEND AWS_PUBLIC_INCLUDES "${AWS_SDKUTILS_DIR}/include/")
+
+
+# Add library.
+add_library(_aws ${AWS_SOURCES})
+
+target_include_directories(_aws SYSTEM BEFORE PUBLIC ${AWS_PUBLIC_INCLUDES})
+target_include_directories(_aws SYSTEM BEFORE PRIVATE ${AWS_PRIVATE_INCLUDES})
+target_compile_definitions(_aws PUBLIC ${AWS_PUBLIC_COMPILE_DEFS})
+target_compile_definitions(_aws PRIVATE ${AWS_PRIVATE_COMPILE_DEFS})
+target_link_libraries(_aws PRIVATE ${AWS_PRIVATE_LIBS})
+
+aws_set_thread_affinity_method(_aws)
+aws_set_thread_name_method(_aws)
+
+# The library is large - avoid bloat.
+if (OMIT_HEAVY_DEBUG_SYMBOLS)
+    target_compile_options (_aws PRIVATE -g0)
+endif()
+
+add_library(ch_contrib::aws_s3 ALIAS _aws)

contrib/aws-crt-cpp

@@ -0,0 +1 @@
+Subproject commit ec0bea288f451d884c0d80d534bc5c66241c39a4

contrib/aws-s2n-tls

@@ -0,0 +1 @@
+Subproject commit 15d534e8a9ca1eda6bacee514e37d08b4f38a526

contrib/aws-s3-cmake/CMakeLists.txt

@@ -1,122 +0,0 @@
-if(NOT OS_FREEBSD)
-    option(ENABLE_S3 "Enable S3" ${ENABLE_LIBRARIES})
-elseif(ENABLE_S3)
-    message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use S3 on FreeBSD")
-endif()
-
-if(NOT ENABLE_S3)
-    message(STATUS "Not using S3")
-    return()
-endif()
-
-SET(AWS_S3_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3")
-SET(AWS_CORE_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-core")
-SET(AWS_CHECKSUMS_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-checksums")
-SET(AWS_COMMON_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-common")
-SET(AWS_EVENT_STREAM_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-event-stream")
-
-OPTION(USE_AWS_MEMORY_MANAGEMENT "Aws memory management" OFF)
-configure_file("${AWS_CORE_LIBRARY_DIR}/include/aws/core/SDKConfig.h.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/include/aws/core/SDKConfig.h" @ONLY)
-
-configure_file("${AWS_COMMON_LIBRARY_DIR}/include/aws/common/config.h.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/include/aws/common/config.h" @ONLY)
-
-
-file(GLOB AWS_CORE_SOURCES
-    "${AWS_CORE_LIBRARY_DIR}/source/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/auth/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/client/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/http/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/http/standard/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/config/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/external/cjson/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/external/tinyxml2/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/internal/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/monitoring/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/net/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/linux-shared/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/platform/linux-shared/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/utils/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/utils/base64/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/utils/event/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/openssl/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/factory/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/utils/json/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/utils/logging/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/utils/memory/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/utils/memory/stl/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/utils/stream/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/utils/threading/*.cpp"
-    "${AWS_CORE_LIBRARY_DIR}/source/utils/xml/*.cpp"
-)
-
-file(GLOB AWS_S3_SOURCES
-    "${AWS_S3_LIBRARY_DIR}/source/*.cpp"
-)
-
-file(GLOB AWS_S3_MODEL_SOURCES
-    "${AWS_S3_LIBRARY_DIR}/source/model/*.cpp"
-)
-
-file(GLOB AWS_EVENT_STREAM_SOURCES
-    "${AWS_EVENT_STREAM_LIBRARY_DIR}/source/*.c"
-)
-
-file(GLOB AWS_COMMON_SOURCES
-    "${AWS_COMMON_LIBRARY_DIR}/source/*.c"
-    "${AWS_COMMON_LIBRARY_DIR}/source/posix/*.c"
-)
-
-file(GLOB AWS_CHECKSUMS_SOURCES
-    "${AWS_CHECKSUMS_LIBRARY_DIR}/source/*.c"
-    "${AWS_CHECKSUMS_LIBRARY_DIR}/source/intel/*.c"
-    "${AWS_CHECKSUMS_LIBRARY_DIR}/source/arm/*.c"
-)
-
-file(GLOB S3_UNIFIED_SRC
-    ${AWS_EVENT_STREAM_SOURCES}
-    ${AWS_COMMON_SOURCES}
-    ${AWS_S3_SOURCES}
-    ${AWS_S3_MODEL_SOURCES}
-    ${AWS_CORE_SOURCES}
-)
-
-set(S3_INCLUDES
-    "${AWS_COMMON_LIBRARY_DIR}/include/"
-    "${AWS_EVENT_STREAM_LIBRARY_DIR}/include/"
-    "${AWS_S3_LIBRARY_DIR}/include/"
-    "${AWS_CORE_LIBRARY_DIR}/include/"
-    "${CMAKE_CURRENT_BINARY_DIR}/include/"
-)
-
-add_library(_aws_s3_checksums ${AWS_CHECKSUMS_SOURCES})
-target_include_directories(_aws_s3_checksums SYSTEM PUBLIC "${AWS_CHECKSUMS_LIBRARY_DIR}/include/")
-if(CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
-    target_compile_definitions(_aws_s3_checksums PRIVATE "-DDEBUG_BUILD")
-endif()
-set_target_properties(_aws_s3_checksums PROPERTIES LINKER_LANGUAGE C)
-set_property(TARGET _aws_s3_checksums PROPERTY C_STANDARD 99)
-
-add_library(_aws_s3 ${S3_UNIFIED_SRC})
-
-target_compile_definitions(_aws_s3 PUBLIC "AWS_SDK_VERSION_MAJOR=1")
-target_compile_definitions(_aws_s3 PUBLIC "AWS_SDK_VERSION_MINOR=7")
-target_compile_definitions(_aws_s3 PUBLIC "AWS_SDK_VERSION_PATCH=231")
-target_include_directories(_aws_s3 SYSTEM BEFORE PUBLIC ${S3_INCLUDES})
-
-if (TARGET OpenSSL::SSL)
-    target_compile_definitions(_aws_s3 PUBLIC -DENABLE_OPENSSL_ENCRYPTION)
-    target_link_libraries(_aws_s3 PRIVATE OpenSSL::Crypto OpenSSL::SSL)
-endif()
-
-target_link_libraries(_aws_s3 PRIVATE _aws_s3_checksums)
-
-# The library is large - avoid bloat.
-if (OMIT_HEAVY_DEBUG_SYMBOLS)
-    target_compile_options (_aws_s3 PRIVATE -g0)
-    target_compile_options (_aws_s3_checksums PRIVATE -g0)
-endif()
-
-add_library(ch_contrib::aws_s3 ALIAS _aws_s3)

contrib/sysroot

@@ -1 +1 @@
-Subproject commit 0f41651860fa4a530ecd68b93a15b8fd77397adf
+Subproject commit f0081b2649b94837855f3bc7d05ef326b100bad8

src/IO/S3/PocoHTTPClient.cpp

@@ -177,7 +177,7 @@ namespace
     bool checkRequestCanReturn2xxAndErrorInBody(Aws::Http::HttpRequest & request)
     {
         auto query_params = request.GetQueryStringParameters();
-        if (request.HasHeader("z-amz-copy-source"))
+        if (request.HasHeader("x-amz-copy-source"))
         {
             /// CopyObject https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html
             if (query_params.empty())

src/IO/S3Common.cpp

@@ -149,13 +149,12 @@ class AWSEC2MetadataClient : public Aws::Internal::AWSHttpResourceClient
     static constexpr char EC2_IMDS_TOKEN_TTL_DEFAULT_VALUE[] = "21600";
     static constexpr char EC2_IMDS_TOKEN_TTL_HEADER[] = "x-aws-ec2-metadata-token-ttl-seconds";
 
-    static constexpr char EC2_DEFAULT_METADATA_ENDPOINT[] = "http://169.254.169.254";
-
 public:
     /// See EC2MetadataClient.
 
-    explicit AWSEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration)
+    explicit AWSEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration, const char * endpoint_)
         : Aws::Internal::AWSHttpResourceClient(client_configuration)
+        , endpoint(endpoint_)
         , logger(&Poco::Logger::get("AWSEC2InstanceProfileConfigLoader"))
     {
     }
@@ -180,7 +179,7 @@ public:
         {
             std::lock_guard locker(token_mutex);
 
-            LOG_TRACE(logger, "Getting default credentials for EC2 instance.");
+            LOG_TRACE(logger, "Getting default credentials for ec2 instance from {}", endpoint);
             auto result = GetResourceWithAWSWebServiceResult(endpoint.c_str(), EC2_SECURITY_CREDENTIALS_RESOURCE, nullptr);
             credentials_string = result.GetPayload();
             if (result.GetResponseCode() == Aws::Http::HttpResponseCode::UNAUTHORIZED)
@@ -286,12 +285,50 @@ public:
     }
 
 private:
-    const Aws::String endpoint = EC2_DEFAULT_METADATA_ENDPOINT;
+    const Aws::String endpoint;
     mutable std::recursive_mutex token_mutex;
     mutable Aws::String token;
     Poco::Logger * logger;
 };
 
+std::shared_ptr<AWSEC2MetadataClient> InitEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration)
+{
+    Aws::String ec2_metadata_service_endpoint = Aws::Environment::GetEnv("AWS_EC2_METADATA_SERVICE_ENDPOINT");
+    auto * logger = &Poco::Logger::get("AWSEC2InstanceProfileConfigLoader");
+    if (ec2_metadata_service_endpoint.empty())
+    {
+        Aws::String ec2_metadata_service_endpoint_mode = Aws::Environment::GetEnv("AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE");
+        if (ec2_metadata_service_endpoint_mode.length() == 0)
+        {
+            ec2_metadata_service_endpoint = "http://169.254.169.254"; //default to IPv4 default endpoint
+        }
+        else
+        {
+            if (ec2_metadata_service_endpoint_mode.length() == 4)
+            {
+                if (Aws::Utils::StringUtils::CaselessCompare(ec2_metadata_service_endpoint_mode.c_str(), "ipv4"))
+                {
+                    ec2_metadata_service_endpoint = "http://169.254.169.254"; //default to IPv4 default endpoint
+                }
+                else if (Aws::Utils::StringUtils::CaselessCompare(ec2_metadata_service_endpoint_mode.c_str(), "ipv6"))
+                {
+                    ec2_metadata_service_endpoint = "http://[fd00:ec2::254]";
+                }
+                else
+                {
+                    LOG_ERROR(logger, "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE can only be set to ipv4 or ipv6, received: {}", ec2_metadata_service_endpoint_mode);
+                }
+            }
+            else
+            {
+                LOG_ERROR(logger, "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE can only be set to ipv4 or ipv6, received: {}", ec2_metadata_service_endpoint_mode);
+            }
+        }
+    }
+    LOG_INFO(logger, "Using IMDS endpoint: {}", ec2_metadata_service_endpoint);
+    return std::make_shared<AWSEC2MetadataClient>(client_configuration, ec2_metadata_service_endpoint.c_str());
+}
+
 class AWSEC2InstanceProfileConfigLoader : public Aws::Config::AWSProfileConfigLoader
 {
 public:
@@ -646,7 +683,7 @@ public:
 
                 aws_client_configuration.retryStrategy = std::make_shared<Aws::Client::DefaultRetryStrategy>(1, 1000);
 
-                auto ec2_metadata_client = std::make_shared<AWSEC2MetadataClient>(aws_client_configuration);
+                auto ec2_metadata_client = InitEC2MetadataClient(aws_client_configuration);
                 auto config_loader = std::make_shared<AWSEC2InstanceProfileConfigLoader>(ec2_metadata_client, !use_insecure_imds_request);
 
                 AddProvider(std::make_shared<AWSInstanceProfileCredentialsProvider>(config_loader));

tests/integration/test_s3_ec2_metadata/configs/use_environment_credentials.xml

@@ -0,0 +1,5 @@
+<clickhouse>
+    <s3>
+        <use_environment_credentials>1</use_environment_credentials>
+    </s3>
+</clickhouse>

tests/integration/test_s3_ec2_metadata/ec2_metadata_server/request_response_server.py

@@ -0,0 +1,36 @@
+import http.server
+import sys
+
+
+class RequestHandler(http.server.BaseHTTPRequestHandler):
+    def get_response(self):
+        if self.path == "/":
+            return "OK"
+        elif self.path == "/latest/meta-data/iam/security-credentials":
+            return "myrole"
+        elif self.path == "/latest/meta-data/iam/security-credentials/myrole":
+            return '{ "Code" : "Success", "Type" : "AWS-HMAC", "AccessKeyId" : "minio", "SecretAccessKey" : "minio123" }'
+        else:
+            return None
+
+    def do_HEAD(self):
+        response = self.get_response()
+        if response:
+            self.send_response(200)
+            self.send_header("Content-Type", "text/plain")
+            self.send_header("Content-Length", len(response.encode()))
+            self.end_headers()
+        else:
+            self.send_response(404)
+            self.send_header("Content-Type", "text/plain")
+            self.end_headers()
+
+    def do_GET(self):
+        self.do_HEAD()
+        response = self.get_response()
+        if response:
+            self.wfile.write(response.encode())
+
+
+httpd = http.server.HTTPServer(("0.0.0.0", int(sys.argv[1])), RequestHandler)
+httpd.serve_forever()

tests/integration/test_s3_ec2_metadata/test.py

@@ -0,0 +1,94 @@
+import pytest
+from helpers.cluster import ClickHouseCluster
+import logging
+import os
+import time
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+EC2_METADATA_SERVER_HOSTNAME = "resolver"
+EC2_METADATA_SERVER_PORT = 8080
+
+cluster = ClickHouseCluster(__file__)
+
+node = cluster.add_instance(
+    "node",
+    with_minio=True,
+    main_configs=["configs/use_environment_credentials.xml"],
+    env_variables={
+        "AWS_EC2_METADATA_SERVICE_ENDPOINT": f"{EC2_METADATA_SERVER_HOSTNAME}:{EC2_METADATA_SERVER_PORT}",
+    },
+)
+
+
+def start_ec2_metadata_server():
+    logging.info("Starting EC2 metadata server")
+    container_id = cluster.get_container_id("resolver")
+
+    cluster.copy_file_to_container(
+        container_id,
+        os.path.join(SCRIPT_DIR, "ec2_metadata_server/request_response_server.py"),
+        "request_response_server.py",
+    )
+
+    cluster.exec_in_container(
+        container_id,
+        ["python", "request_response_server.py", str(EC2_METADATA_SERVER_PORT)],
+        detach=True,
+    )
+
+    # Wait for the server to start.
+    num_attempts = 100
+    for attempt in range(num_attempts):
+        ping_response = cluster.exec_in_container(
+            container_id,
+            ["curl", "-s", f"http://localhost:{EC2_METADATA_SERVER_PORT}/"],
+            nothrow=True,
+        )
+        if ping_response != "OK":
+            if attempt == num_attempts - 1:
+                assert ping_response == "OK", 'Expected "OK", but got "{}"'.format(
+                    ping_response
+                )
+            else:
+                time.sleep(1)
+        else:
+            logging.debug(
+                f"request_response_server.py answered {ping_response} on attempt {attempt}"
+            )
+            break
+
+    logging.info("EC2 metadata server started")
+
+
+@pytest.fixture(scope="module", autouse=True)
+def start_cluster():
+    try:
+        cluster.start()
+        start_ec2_metadata_server()
+        yield
+    finally:
+        cluster.shutdown()
+
+
+def test_credentials_from_ec2_metadata():
+    node.query(
+        f"INSERT INTO FUNCTION s3('http://{cluster.minio_host}:{cluster.minio_port}/{cluster.minio_bucket}/test1.jsonl') SELECT * FROM numbers(100)"
+    )
+
+    assert (
+        "100"
+        == node.query(
+            f"SELECT count() FROM s3('http://{cluster.minio_host}:{cluster.minio_port}/{cluster.minio_bucket}/test1.jsonl')"
+        ).strip()
+    )
+
+    expected_logs = [
+        "Getting default credentials for ec2 instance from resolver:8080",
+        "Calling EC2MetadataService resource, /latest/meta-data/iam/security-credentials returned credential string myrole",
+        "Calling EC2MetadataService resource /latest/meta-data/iam/security-credentials/myrole",
+        "Successfully pulled credentials from EC2MetadataService with access key",
+    ]
+
+    for expected_msg in expected_logs:
+        node.contains_in_log("AWSEC2InstanceProfileConfigLoader: " + expected_msg)