Merge branch 'master' into round-robin-merge-scheduler

2024-11-22 07:31:57 +00:00 · 2023-02-14 20:39:02 +01:00 · 2023-02-14 20:39:02 +01:00 · b7a4fa8bd9
commit b7a4fa8bd9
parent a7bbf02bac e914e7c98d
213 changed files with 4810 additions and 1409 deletions
--- a/.github/workflows/debug.yml
+++ b/.github/workflows/debug.yml
@ -2,7 +2,7 @@
 name: Debug

 'on':
-  [push, pull_request, release, workflow_dispatch, workflow_call]
+  [push, pull_request, pull_request_review, release, workflow_dispatch, workflow_call]

 jobs:
  DebugInfo:
--- a/.github/workflows/pull_request_approved.yml
+++ b/.github/workflows/pull_request_approved.yml
@ -0,0 +1,23 @@
+name: PullRequestApprovedCI
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+on:  # yamllint disable-line rule:truthy
+  pull_request_review:
+    types:
+      - submitted
+
+jobs:
+  MergeOnApproval:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Merge approved PR
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 merge_pr.py --check-approved
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -476,7 +476,12 @@ enable_testing() # Enable for tests without binary

 option(ENABLE_OPENSSL "This option performs a build with OpenSSL. NOTE! This option is insecure and should never be used. By default, ClickHouse uses and only supports BoringSSL" OFF)

-option(ENABLE_OPENSSL_DYNAMIC "This option removes SSL from ClickHouse and will link to the OpenSSL version supplied by OS." OFF)
+if (ARCH_S390X)
+    set(ENABLE_OPENSSL_DYNAMIC_DEFAULT ON)
+else ()
+    set(ENABLE_OPENSSL_DYNAMIC_DEFAULT OFF)
+endif ()
+option(ENABLE_OPENSSL_DYNAMIC "This option removes SSL from ClickHouse and will link to the OpenSSL version supplied by OS." ${ENABLE_OPENSSL_DYNAMIC_DEFAULT})

 # when installing to /usr - place configs to /etc but for /usr/local place to /usr/local/etc
 if (CMAKE_INSTALL_PREFIX STREQUAL "/usr")
--- a/PreLoad.cmake
+++ b/PreLoad.cmake
@ -84,7 +84,10 @@ if (OS MATCHES "Linux"
        set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "")
    elseif (ARCH MATCHES "^(ppc64le.*|PPC64LE.*)")
        set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-ppc64le.cmake" CACHE INTERNAL "")
+    elseif (ARCH MATCHES "^(s390x.*|S390X.*)")
+        set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-s390x.cmake" CACHE INTERNAL "")
    else ()
        message (FATAL_ERROR "Unsupported architecture: ${ARCH}")
    endif ()
+
 endif()
--- a/base/base/defines.h
+++ b/base/base/defines.h
@ -28,8 +28,8 @@
 #define NO_INLINE __attribute__((__noinline__))
 #define MAY_ALIAS __attribute__((__may_alias__))

-#if !defined(__x86_64__) && !defined(__aarch64__) && !defined(__PPC__) && !(defined(__riscv) && (__riscv_xlen == 64))
-#    error "The only supported platforms are x86_64 and AArch64, PowerPC (work in progress) and RISC-V 64 (experimental)"
+#if !defined(__x86_64__) && !defined(__aarch64__) && !defined(__PPC__) && !defined(__s390x__) && !(defined(__riscv) && (__riscv_xlen == 64))
+#    error "The only supported platforms are x86_64 and AArch64, PowerPC (work in progress), s390x (work in progress) and RISC-V 64 (experimental)"
 #endif

 /// Check for presence of address sanitizer
--- a/base/base/wide_integer_impl.h
+++ b/base/base/wide_integer_impl.h
@ -360,7 +360,7 @@ struct integer<Bits, Signed>::_impl
        constexpr const unsigned to_copy = min_bits / base_bits;

        for (unsigned i = 0; i < to_copy; ++i)
-            self.items[little(i)] = rhs.items[little(i)];
+            self.items[little(i)] = rhs.items[integer<Bits2, Signed2>::_impl::little(i)];

        if constexpr (Bits > Bits2)
        {
--- a/base/poco/Util/src/SystemConfiguration.cpp
+++ b/base/poco/Util/src/SystemConfiguration.cpp
@ -89,7 +89,9 @@ bool SystemConfiguration::getRaw(const std::string& key, std::string& value) con
 			Poco::Environment::NodeId id;
 			Poco::Environment::nodeId(id);
 			char result[13];
-			std::sprintf(result, "%02x%02x%02x%02x%02x%02x",
+			std::snprintf(result, 
+                sizeof(result),
+                "%02x%02x%02x%02x%02x%02x",
 				id[0],
 				id[1],
 				id[2],
--- a/cmake/arch.cmake
+++ b/cmake/arch.cmake
@ -7,6 +7,8 @@ elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)")
    set (ARCH_AARCH64 1)
 elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc64le.*|ppc64le.*|PPC64LE.*)")
    set (ARCH_PPC64LE 1)
+elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(s390x.*|S390X.*)")
+    set (ARCH_S390X 1)
 elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")
    set (ARCH_RISCV64 1)
 else ()
--- a/cmake/linux/toolchain-s390x.cmake
+++ b/cmake/linux/toolchain-s390x.cmake
@ -0,0 +1,28 @@
+# See linux/toolchain-x86_64.cmake for details about multiple load of toolchain file.
+include_guard(GLOBAL)
+
+set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+
+set (CMAKE_SYSTEM_NAME "Linux")
+set (CMAKE_SYSTEM_PROCESSOR "s390x")
+set (CMAKE_C_COMPILER_TARGET "s390x-linux-gnu")
+set (CMAKE_CXX_COMPILER_TARGET "s390x-linux-gnu")
+set (CMAKE_ASM_COMPILER_TARGET "s390x-linux-gnu")
+
+# Will be changed later, but somehow needed to be set here.
+set (CMAKE_AR "ar")
+set (CMAKE_RANLIB "ranlib")
+
+set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-s390x")
+
+set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/s390x-linux-gnu/libc")
+
+set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+
+set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
+
+set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
--- a/cmake/tools.cmake
+++ b/cmake/tools.cmake
@ -53,20 +53,23 @@ list (GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR)
 # Example values: `lld-10`, `gold`.
 option (LINKER_NAME "Linker name or full path")

-if (NOT LINKER_NAME)
-    if (COMPILER_GCC)
-        find_program (LLD_PATH NAMES "ld.lld")
-        find_program (GOLD_PATH NAMES "ld.gold")
-    elseif (COMPILER_CLANG)
-        # llvm lld is a generic driver.
-        # Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-ld (WebAssembly) instead
-        if (OS_LINUX)
-            find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "ld.lld")
-        elseif (OS_DARWIN)
-            find_program (LLD_PATH NAMES "ld64.lld-${COMPILER_VERSION_MAJOR}" "ld64.lld")
+# s390x doesnt support lld
+if (NOT ARCH_S390X)
+    if (NOT LINKER_NAME)
+        if (COMPILER_GCC)
+            find_program (LLD_PATH NAMES "ld.lld")
+            find_program (GOLD_PATH NAMES "ld.gold")
+        elseif (COMPILER_CLANG)
+            # llvm lld is a generic driver.
+            # Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-ld (WebAssembly) instead
+            if (OS_LINUX)
+                find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "ld.lld")
+            elseif (OS_DARWIN)
+                find_program (LLD_PATH NAMES "ld64.lld-${COMPILER_VERSION_MAJOR}" "ld64.lld")
+            endif ()
+            find_program (GOLD_PATH NAMES "ld.gold" "gold")
        endif ()
-        find_program (GOLD_PATH NAMES "ld.gold" "gold")
-    endif ()
+    endif()
 endif()

 if ((OS_LINUX OR OS_DARWIN) AND NOT LINKER_NAME)
--- a/contrib/boost-cmake/CMakeLists.txt
+++ b/contrib/boost-cmake/CMakeLists.txt
@ -126,6 +126,12 @@ elseif (ARCH_PPC64LE)
        "${LIBRARY_DIR}/libs/context/src/asm/make_ppc64_sysv_elf_gas.S"
        "${LIBRARY_DIR}/libs/context/src/asm/ontop_ppc64_sysv_elf_gas.S"
    )
+elseif (ARCH_S390X)
+    set (SRCS_CONTEXT ${SRCS_CONTEXT}
+        "${LIBRARY_DIR}/libs/context/src/asm/jump_s390x_sysv_elf_gas.S"
+        "${LIBRARY_DIR}/libs/context/src/asm/make_s390x_sysv_elf_gas.S"
+        "${LIBRARY_DIR}/libs/context/src/asm/ontop_s390x_sysv_elf_gas.S"
+    )
 elseif (ARCH_RISCV64)
    set (SRCS_CONTEXT ${SRCS_CONTEXT}
        "${LIBRARY_DIR}/libs/context/src/asm/jump_riscv64_sysv_elf_gas.S"
--- a/contrib/cityhash102/src/config.h
+++ b/contrib/cityhash102/src/config.h
@ -68,7 +68,7 @@

 /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
   significant byte first (like Motorola and SPARC, unlike Intel). */
-#if defined AC_APPLE_UNIVERSAL_BUILD
+#if defined AC_APPLE_UNIVERSAL_BUILD || defined(__s390x__)
 # if defined __BIG_ENDIAN__
 #  define WORDS_BIGENDIAN 1
 # endif
--- a/contrib/libhdfs3-cmake/CMake/Options.cmake
+++ b/contrib/libhdfs3-cmake/CMake/Options.cmake
@ -7,9 +7,9 @@ CHECK_FUNCTION_EXISTS(nanosleep HAVE_NANOSLEEP)
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-strict-aliasing")
 SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing")

-IF(ENABLE_SSE STREQUAL ON AND NOT ARCH_PPC64LE AND NOT ARCH_AARCH64 AND NOT ARCH_AARCH64)
+IF(ENABLE_SSE STREQUAL ON AND ARCH_AMD64)
    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
-ENDIF(ENABLE_SSE STREQUAL ON AND NOT ARCH_PPC64LE AND NOT ARCH_AARCH64 AND NOT ARCH_AARCH64)
+ENDIF()

 IF(NOT TEST_HDFS_PREFIX)
 SET(TEST_HDFS_PREFIX "./" CACHE STRING "default directory prefix used for test." FORCE)
--- a/contrib/libhdfs3-cmake/CMakeLists.txt
+++ b/contrib/libhdfs3-cmake/CMakeLists.txt
@ -1,4 +1,4 @@
-if(NOT ARCH_AARCH64 AND NOT OS_FREEBSD AND NOT APPLE)
+if(NOT ARCH_AARCH64 AND NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE AND NOT ARCH_S390X)
    option(ENABLE_HDFS "Enable HDFS" ${ENABLE_LIBRARIES})
 elseif(ENABLE_HDFS)
    message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use HDFS3 with current configuration")
@ -106,7 +106,7 @@ set(SRCS
 if (ARCH_AMD64)
    find_program(YASM_PATH NAMES yasm)
    if (NOT YASM_PATH)
-        message(FATAL_ERROR "Please install the Yasm assembler")
+        message(FATAL_ERROR "Please install the Yasm assembler to build ClickHouse with Hadoop Distributed File System (HDFS) support")
    endif ()
    add_custom_command(
        OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/crc_iscsi_v_pcl.o
--- a/contrib/librdkafka-cmake/config.h.in
+++ b/contrib/librdkafka-cmake/config.h.in
@ -66,7 +66,7 @@
 #cmakedefine WITH_SASL_OAUTHBEARER 1
 #cmakedefine WITH_SASL_CYRUS 1
 // crc32chw
-#if !defined(__PPC__) && !defined(__riscv) && !defined(__aarch64__)
+#if !defined(__PPC__) && !defined(__riscv) && !defined(__aarch64__) && !defined(__s390x__)
 #define WITH_CRC32C_HW 1
 #endif
 // regex
--- a/contrib/openldap-cmake/CMakeLists.txt
+++ b/contrib/openldap-cmake/CMakeLists.txt
@ -19,6 +19,7 @@ if (NOT(
    ( "${_system_name}" STREQUAL "linux"   AND "${_system_processor}" STREQUAL "x86_64"  ) OR
    ( "${_system_name}" STREQUAL "linux"   AND "${_system_processor}" STREQUAL "aarch64" ) OR
    ( "${_system_name}" STREQUAL "linux"   AND "${_system_processor}" STREQUAL "ppc64le" ) OR
+    ( "${_system_name}" STREQUAL "linux"   AND "${_system_processor}" STREQUAL "s390x"   ) OR
    ( "${_system_name}" STREQUAL "freebsd" AND "${_system_processor}" STREQUAL "x86_64"  ) OR
    ( "${_system_name}" STREQUAL "freebsd" AND "${_system_processor}" STREQUAL "aarch64" ) OR
    ( "${_system_name}" STREQUAL "darwin"  AND "${_system_processor}" STREQUAL "x86_64"  ) OR
--- a/contrib/openldap-cmake/linux_s390x/include/lber_types.h
+++ b/contrib/openldap-cmake/linux_s390x/include/lber_types.h
@ -0,0 +1,63 @@
+/* include/lber_types.h.  Generated from lber_types.hin by configure.  */
+/* $OpenLDAP$ */
+/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
+ *
+ * Copyright 1998-2020 The OpenLDAP Foundation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+/*
+ * LBER types
+ */
+
+#ifndef _LBER_TYPES_H
+#define _LBER_TYPES_H
+
+#include <ldap_cdefs.h>
+
+LDAP_BEGIN_DECL
+
+/* LBER boolean, enum, integers (32 bits or larger) */
+#define LBER_INT_T int
+
+/* LBER tags (32 bits or larger) */
+#define LBER_TAG_T long
+
+/* LBER socket descriptor */
+#define LBER_SOCKET_T int
+
+/* LBER lengths (32 bits or larger) */
+#define LBER_LEN_T long
+
+/* ------------------------------------------------------------ */
+
+/* booleans, enumerations, and integers */
+typedef LBER_INT_T ber_int_t;
+
+/* signed and unsigned versions */
+typedef signed LBER_INT_T ber_sint_t;
+typedef unsigned LBER_INT_T ber_uint_t;
+
+/* tags */
+typedef unsigned LBER_TAG_T ber_tag_t;
+
+/* "socket" descriptors */
+typedef LBER_SOCKET_T ber_socket_t;
+
+/* lengths */
+typedef unsigned LBER_LEN_T ber_len_t;
+
+/* signed lengths */
+typedef signed LBER_LEN_T ber_slen_t;
+
+LDAP_END_DECL
+
+#endif /* _LBER_TYPES_H */
--- a/contrib/openldap-cmake/linux_s390x/include/ldap_config.h
+++ b/contrib/openldap-cmake/linux_s390x/include/ldap_config.h
@ -0,0 +1,74 @@
+/* Generated from ./ldap_config.hin on Thu Mar 31 05:25:33 UTC 2022 */
+/* $OpenLDAP$ */
+/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
+ *
+ * Copyright 1998-2020 The OpenLDAP Foundation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+/*
+ * This file works in conjunction with OpenLDAP configure system.
+ * If you do no like the values below, adjust your configure options.
+ */
+
+#ifndef _LDAP_CONFIG_H
+#define _LDAP_CONFIG_H
+
+/* directory separator */
+#ifndef LDAP_DIRSEP
+#ifndef _WIN32
+#define LDAP_DIRSEP "/"
+#else
+#define LDAP_DIRSEP "\\"
+#endif
+#endif
+
+/* directory for temporary files */
+#if defined(_WIN32)
+# define LDAP_TMPDIR "C:\\."	/* we don't have much of a choice */
+#elif defined( _P_tmpdir )
+# define LDAP_TMPDIR _P_tmpdir
+#elif defined( P_tmpdir )
+# define LDAP_TMPDIR P_tmpdir
+#elif defined( _PATH_TMPDIR )
+# define LDAP_TMPDIR _PATH_TMPDIR
+#else
+# define LDAP_TMPDIR LDAP_DIRSEP "tmp"
+#endif
+
+/* directories */
+#ifndef LDAP_BINDIR
+#define LDAP_BINDIR			"/usr/local/bin"
+#endif
+#ifndef LDAP_SBINDIR
+#define LDAP_SBINDIR		"/usr/local/sbin"
+#endif
+#ifndef LDAP_DATADIR
+#define LDAP_DATADIR		"/usr/local/share/openldap"
+#endif
+#ifndef LDAP_SYSCONFDIR
+#define LDAP_SYSCONFDIR		"/usr/local/etc/openldap"
+#endif
+#ifndef LDAP_LIBEXECDIR
+#define LDAP_LIBEXECDIR		"/usr/local/libexec"
+#endif
+#ifndef LDAP_MODULEDIR
+#define LDAP_MODULEDIR		"/usr/local/libexec/openldap"
+#endif
+#ifndef LDAP_RUNDIR
+#define LDAP_RUNDIR			"/usr/local/var"
+#endif
+#ifndef LDAP_LOCALEDIR
+#define LDAP_LOCALEDIR		""
+#endif
+
+
+#endif /* _LDAP_CONFIG_H */
--- a/contrib/openldap-cmake/linux_s390x/include/ldap_features.h
+++ b/contrib/openldap-cmake/linux_s390x/include/ldap_features.h
@ -0,0 +1,56 @@
+/* include/ldap_features.h.  Generated from ldap_features.hin by configure.  */
+/* $OpenLDAP$ */
+/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
+ *
+ * Copyright 1998-2020 The OpenLDAP Foundation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+/* 
+ * LDAP Features
+ */
+
+#ifndef _LDAP_FEATURES_H
+#define _LDAP_FEATURES_H 1
+
+/* OpenLDAP API version macros */
+#define LDAP_VENDOR_VERSION 000000
+#define LDAP_VENDOR_VERSION_MAJOR 2
+#define LDAP_VENDOR_VERSION_MINOR X
+#define LDAP_VENDOR_VERSION_PATCH X
+
+/*
+** WORK IN PROGRESS!
+**
+** OpenLDAP reentrancy/thread-safeness should be dynamically
+** checked using ldap_get_option().
+**
+** If built with thread support, the -lldap implementation is:
+**		LDAP_API_FEATURE_THREAD_SAFE (basic thread safety)
+**		LDAP_API_FEATURE_SESSION_THREAD_SAFE
+**		LDAP_API_FEATURE_OPERATION_THREAD_SAFE
+**
+** The preprocessor flag LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE
+** can be used to determine if -lldap is thread safe at compile
+** time.
+**
+*/
+
+/* is -lldap reentrant or not */
+#define LDAP_API_FEATURE_X_OPENLDAP_REENTRANT 1
+
+/* is -lldap thread safe or not */
+#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1
+
+/* LDAP v2 Referrals */
+/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */
+
+#endif /* LDAP_FEATURES */
--- a/contrib/openldap-cmake/linux_s390x/include/portable.h
+++ b/contrib/openldap-cmake/linux_s390x/include/portable.h
--- a/contrib/rocksdb
+++ b/contrib/rocksdb
@ -1 +1 @@
-Subproject commit 2c8998e26c6d46b27c710d7829c3a15e34959f70
+Subproject commit 66e3cbec31400ed3a23deb878c5d7f56f990f0ae
--- a/contrib/s2geometry
+++ b/contrib/s2geometry
@ -1 +1 @@
-Subproject commit 471fe9dc931a4bb560333545186e9b5da168ac83
+Subproject commit 4a7ebd5da04cb6c9ea38bbf5914a9f8f3c768564
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@ -1,7 +1,47 @@
-# rebuild in #33610
 # docker build -t clickhouse/binary-builder .
 ARG FROM_TAG=latest
+FROM clickhouse/test-util:latest AS cctools
+# The cctools are built always from the clickhouse/test-util:latest and cached inline
+# Theoretically, it should improve rebuild speed significantly
+ENV CC=clang-${LLVM_VERSION}
+ENV CXX=clang++-${LLVM_VERSION}
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+# DO NOT PUT ANYTHING BEFORE THREE NEXT `RUN` DIRECTIVES
+# THE MOST HEAVY OPERATION MUST BE THE FIRST IN THE CACHE
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+# libtapi is required to support .tbh format from recent MacOS SDKs
+RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \
+    && cd apple-libtapi \
+    && INSTALLPREFIX=/cctools ./build.sh \
+    && ./install.sh \
+    && cd .. \
+    && rm -rf apple-libtapi
+
+# Build and install tools for cross-linking to Darwin (x86-64)
+RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \
+    && cd cctools-port/cctools \
+    && ./configure --prefix=/cctools --with-libtapi=/cctools \
+        --target=x86_64-apple-darwin \
+    && make install -j$(nproc) \
+    && cd ../.. \
+    && rm -rf cctools-port
+
+# Build and install tools for cross-linking to Darwin (aarch64)
+RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \
+    && cd cctools-port/cctools \
+    && ./configure --prefix=/cctools --with-libtapi=/cctools \
+        --target=aarch64-apple-darwin \
+    && make install -j$(nproc) \
+    && cd ../.. \
+    && rm -rf cctools-port
+
+# !!!!!!!!!!!
+# END COMPILE
+# !!!!!!!!!!!
+
 FROM clickhouse/test-util:$FROM_TAG
+ENV CC=clang-${LLVM_VERSION}
+ENV CXX=clang++-${LLVM_VERSION}

 # Rust toolchain and libraries
 ENV RUSTUP_HOME=/rust/rustup
@ -16,80 +56,53 @@ RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \
    rustup target add aarch64-apple-darwin && \
    rustup target add powerpc64le-unknown-linux-gnu

-RUN apt-get update && \
-    apt-get install --yes \
-        gcc-aarch64-linux-gnu \
+# NOTE: Seems like gcc-11 is too new for ubuntu20 repository
+# A cross-linker for RISC-V 64 (we need it, because LLVM's LLD does not work):
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
+    && apt-get update \
+    && apt-get install --yes \
+        binutils-riscv64-linux-gnu \
        build-essential \
+        g++-11 \
+        gcc-11 \
+        gcc-aarch64-linux-gnu \
        libc6 \
        libc6-dev \
        libc6-dev-arm64-cross \
        yasm \
-        zstd && \
-    apt-get clean
-
-ENV CC=clang-${LLVM_VERSION}
-ENV CXX=clang++-${LLVM_VERSION}
-
-# libtapi is required to support .tbh format from recent MacOS SDKs
-RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \
-    && cd apple-libtapi \
-    && INSTALLPREFIX=/cctools ./build.sh \
-    && ./install.sh \
-    && cd .. \
-    && rm -rf apple-libtapi
-
-# Build and install tools for cross-linking to Darwin (x86-64)
-RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \
-    && cd cctools-port/cctools \
-    && ./configure --prefix=/cctools --with-libtapi=/cctools \
-        --target=x86_64-apple-darwin \
-    && make install \
-    && cd ../.. \
-    && rm -rf cctools-port
-
-# Build and install tools for cross-linking to Darwin (aarch64)
-RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \
-    && cd cctools-port/cctools \
-    && ./configure --prefix=/cctools --with-libtapi=/cctools \
-        --target=aarch64-apple-darwin \
-    && make install \
-    && cd ../.. \
-    && rm -rf cctools-port
+        zstd \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists

 # Download toolchain and SDK for Darwin
 RUN wget -nv https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz

-# NOTE: Seems like gcc-11 is too new for ubuntu20 repository
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
-    && apt-get update \
-    && apt-get install gcc-11 g++-11 --yes \
-    && apt-get clean
-
-# A cross-linker for RISC-V 64 (we need it, because LLVM's LLD does not work):
-RUN apt-get install binutils-riscv64-linux-gnu
-
 # Architecture of the image when BuildKit/buildx is used
 ARG TARGETARCH
 ARG NFPM_VERSION=2.20.0

 RUN arch=${TARGETARCH:-amd64} \
-  && curl -Lo /tmp/nfpm.deb "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${arch}.deb" \
-  && dpkg -i /tmp/nfpm.deb \
-  && rm /tmp/nfpm.deb
+    && curl -Lo /tmp/nfpm.deb "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${arch}.deb" \
+    && dpkg -i /tmp/nfpm.deb \
+    && rm /tmp/nfpm.deb

-ARG GO_VERSION=1.18.3
+ARG GO_VERSION=1.19.5
 # We need go for clickhouse-diagnostics
 RUN arch=${TARGETARCH:-amd64} \
-  && curl -Lo /tmp/go.tgz "https://go.dev/dl/go${GO_VERSION}.linux-${arch}.tar.gz" \
-  && tar -xzf /tmp/go.tgz -C /usr/local/ \
-  && rm /tmp/go.tgz
+    && curl -Lo /tmp/go.tgz "https://go.dev/dl/go${GO_VERSION}.linux-${arch}.tar.gz" \
+    && tar -xzf /tmp/go.tgz -C /usr/local/ \
+    && rm /tmp/go.tgz

 ENV PATH="$PATH:/usr/local/go/bin"
 ENV GOPATH=/workdir/go
 ENV GOCACHE=/workdir/

-RUN curl https://raw.githubusercontent.com/matus-chochlik/ctcache/7fd516e91c17779cbc6fc18bd119313d9532dd90/clang-tidy-cache -Lo /usr/bin/clang-tidy-cache \
-  && chmod +x /usr/bin/clang-tidy-cache
+ARG CLANG_TIDY_SHA1=03644275e794b0587849bfc2ec6123d5ae0bdb1c
+RUN curl -Lo /usr/bin/clang-tidy-cache \
+        "https://raw.githubusercontent.com/matus-chochlik/ctcache/$CLANG_TIDY_SHA1/clang-tidy-cache" \
+    && chmod +x /usr/bin/clang-tidy-cache
+
+COPY --from=cctools /cctools /cctools

 RUN mkdir /workdir && chmod 777 /workdir
 WORKDIR /workdir
--- a/docker/server/README.md
+++ b/docker/server/README.md
@ -2,23 +2,30 @@

 ## What is ClickHouse?

-We are the creators of the popular open-source column-oriented DBMS (columnar database management system) for online analytical processing (OLAP) which allows users to generate analytical reports using SQL queries in real-time.
+ClickHouse is an open-source column-oriented DBMS (columnar database management system) for online analytical processing (OLAP) that allows users to generate analytical reports using SQL queries in real-time.

 ClickHouse works 100-1000x faster than traditional database management systems, and processes hundreds of millions to over a billion rows and tens of gigabytes of data per server per second.  With a widespread user base around the globe, the technology has received praise for its reliability, ease of use, and fault tolerance.

 For more information and documentation see https://clickhouse.com/.

+## Versions
+
+- The `latest` tag points to the latest release of the latest stable branch.
+- Branch tags like `22.2` point to the latest release of the corresponding branch.
+- Full version tags like `22.2.3.5` point to the corresponding release.
+- The tag `head` is built from the latest commit to the default branch.
+- Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`.
+
 ## How to use this image

 ### start server instance
-
 ```bash
 docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
 ```

 By default, ClickHouse will be accessible only via the Docker network. See the [networking section below](#networking).

-By default, starting above server instance will be run as the `default` user without a password.
+By default, starting above server instance will be run as the `default` user without  password.

 ### connect to it from a native client

@ -36,7 +43,7 @@ More information about the [ClickHouse client](https://clickhouse.com/docs/en/in
 echo "SELECT 'Hello, ClickHouse!'" | docker run -i --rm --link some-clickhouse-server:clickhouse-server curlimages/curl 'http://clickhouse-server:8123/?query=' -s --data-binary @-
 ```

-More information about [ClickHouse HTTP Interface](https://clickhouse.com/docs/en/interfaces/http/).
+More information about the [ClickHouse HTTP Interface](https://clickhouse.com/docs/en/interfaces/http/).

 ### stopping / removing the container

@ -54,7 +61,7 @@ docker run -d -p 18123:8123 -p19000:9000 --name some-clickhouse-server --ulimit
 echo 'SELECT version()' | curl 'http://localhost:18123/' --data-binary @-
 ```

-```response
+```
 22.6.3.35
 ```

@ -65,7 +72,7 @@ docker run -d --network=host --name some-clickhouse-server --ulimit nofile=26214
 echo 'SELECT version()' | curl 'http://localhost:8123/' --data-binary @-
 ```

-```response
+```
 22.6.3.35
 ```

@ -93,7 +100,7 @@ You may also want to mount:

 ClickHouse has some advanced functionality, which requires enabling several [Linux capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html).

-These are optional and can be enabled using the following [docker command-line arguments](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities):
+They are optional and can be enabled using the following [docker command-line arguments](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities):

 ```bash
 docker run -d \
@ -113,7 +120,7 @@ ClickHouse configuration is represented with a file "config.xml" ([documentation
 docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /path/to/your/config.xml:/etc/clickhouse-server/config.xml clickhouse/clickhouse-server
 ```

-### Start server as a custom user
+### Start server as custom user

 ```bash
 # $(pwd)/data/clickhouse should exist and be owned by current user
@ -122,7 +129,7 @@ docker run --rm --user ${UID}:${GID} --name some-clickhouse-server --ulimit nofi

 When you use the image with local directories mounted, you probably want to specify the user to maintain the proper file ownership. Use the `--user` argument and mount `/var/lib/clickhouse` and `/var/log/clickhouse-server` inside the container. Otherwise, the image will complain and not start.

-### Start server from root (useful in case of userns enabled)
+### Start server from root (useful in case of enabled user namespace)

 ```bash
 docker run --rm -e CLICKHOUSE_UID=0 -e CLICKHOUSE_GID=0 --name clickhouse-server-userns -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server
@ -130,7 +137,7 @@ docker run --rm -e CLICKHOUSE_UID=0 -e CLICKHOUSE_GID=0 --name clickhouse-server

 ### How to create default database and user on starting

-Sometimes you may want to create a user (user named `default` is used by default) and database on image start. You can do it using environment variables `CLICKHOUSE_DB`, `CLICKHOUSE_USER`, `CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT` and `CLICKHOUSE_PASSWORD`:
+Sometimes you may want to create a user (user named `default` is used by default) and database on a container start. You can do it using environment variables `CLICKHOUSE_DB`, `CLICKHOUSE_USER`, `CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT` and `CLICKHOUSE_PASSWORD`:

 ```bash
 docker run --rm -e CLICKHOUSE_DB=my_database -e CLICKHOUSE_USER=username -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -e CLICKHOUSE_PASSWORD=password -p 9000:9000/tcp clickhouse/clickhouse-server
@ -156,4 +163,3 @@ EOSQL
 ## License

 View [license information](https://github.com/ClickHouse/ClickHouse/blob/master/LICENSE) for the software contained in this image.
-
--- a/docker/test/performance-comparison/perf.py
+++ b/docker/test/performance-comparison/perf.py
@ -26,6 +26,22 @@ logging.basicConfig(
 total_start_seconds = time.perf_counter()
 stage_start_seconds = total_start_seconds

+# Thread executor that does not hides exception that happens during function
+# execution, and rethrows it after join()
+class SafeThread(Thread):
+    run_exception = None
+
+    def run(self):
+        try:
+            super().run()
+        except:
+            self.run_exception = sys.exc_info()
+
+    def join(self):
+        super().join()
+        if self.run_exception:
+            raise self.run_exception[1]
+

 def reportStageEnd(stage):
    global stage_start_seconds, total_start_seconds
@ -283,7 +299,7 @@ if not args.use_existing_tables:
            print(f"create\t{index}\t{connection.last_query.elapsed}\t{tsv_escape(q)}")

    threads = [
-        Thread(target=do_create, args=(connection, index, create_queries))
+        SafeThread(target=do_create, args=(connection, index, create_queries))
        for index, connection in enumerate(all_connections)
    ]

--- a/docker/test/stateful/run.sh
+++ b/docker/test/stateful/run.sh
@ -155,7 +155,7 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
    sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||:
 fi

-rg -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||:
+rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server.log ||:

 zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst ||:
 # FIXME: remove once only github actions will be left
@ -166,8 +166,8 @@ if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
    tar --zstd -c -h -f /test_output/clickhouse_coverage.tar.zst /profraw ||:
 fi
 if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
-    rg -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server1.log ||:
-    rg -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server2.log ||:
+    rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server1.log ||:
+    rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server2.log ||:
    zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.zst ||:
    zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.zst ||:
    # FIXME: remove once only github actions will be left
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@ -169,7 +169,7 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
    sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||:
 fi

-rg -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||:
+rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server.log ||:
 zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst &

 # Compress tables.
@ -215,8 +215,8 @@ fi
 tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:

 if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
-    rg -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server1.log ||:
-    rg -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server2.log ||:
+    rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server1.log ||:
+    rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server2.log ||:
    zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.zst ||:
    zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.zst ||:
    # FIXME: remove once only github actions will be left
--- a/docker/test/stress/run.sh
+++ b/docker/test/stress/run.sh
@ -546,6 +546,9 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
        # it uses recently introduced settings which previous versions may not have
        rm -f /etc/clickhouse-server/users.d/insert_keeper_retries.xml ||:

+        # Turn on after 23.1
+        rm -f /etc/clickhouse-server/users.d/prefetch_settings.xml ||:
+
        start

        clickhouse-client --query="SELECT 'Server version: ', version()"
@ -718,7 +721,7 @@ mv /var/log/clickhouse-server/stderr.log /test_output/

 # Write check result into check_status.tsv
 # Try to choose most specific error for the whole check status
-clickhouse-local --structure "test String, res String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' order by
+clickhouse-local --structure "test String, res String, time Nullable(Float32), desc String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' order by
 (test like 'Backward compatibility check%'),  -- BC check goes last
 (test like '%Sanitizer%') DESC,
 (test like '%Killed by signal%') DESC,
@ -732,7 +735,7 @@ clickhouse-local --structure "test String, res String" -q "SELECT 'failure', tes
 (test like '%Error message%') DESC,
 (test like '%previous release%') DESC,
 rowNumberInAllBlocks()
-LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv
+LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv || echo "failure\tCannot parse test_results.tsv" > /test_output/check_status.tsv
 [ -s /test_output/check_status.tsv ] || echo -e "success\tNo errors found" > /test_output/check_status.tsv

 # Core dumps
--- a/docs/en/getting-started/install.md
+++ b/docs/en/getting-started/install.md
@ -1,10 +1,11 @@
 ---
 sidebar_label: Install
 keywords: [clickhouse, install, getting started, quick start]
+description: Install ClickHouse
 slug: /en/install
 ---

-# Installing ClickHouse
+# Install ClickHouse

 You have three options for getting up and running with ClickHouse:

--- a/docs/en/operations/clickhouse-keeper.md
+++ b/docs/en/operations/clickhouse-keeper.md
@ -42,7 +42,7 @@ Internal coordination settings are located in the `<keeper_server>.<coordination
 -    `session_timeout_ms` — Max timeout for client session (ms) (default: 100000).
 -    `dead_session_check_period_ms` — How often ClickHouse Keeper checks for dead sessions and removes them (ms) (default: 500).
 -    `heart_beat_interval_ms` — How often a ClickHouse Keeper leader will send heartbeats to followers (ms) (default: 500).
-    `election_timeout_lower_bound_ms` — If the follower does not receive a heartbeat from the leader in this interval, then it can initiate leader election (default: 1000).
+-    `election_timeout_lower_bound_ms` — If the follower does not receive a heartbeat from the leader in this interval, then it can initiate leader election (default: 1000). Must be less than or equal to `election_timeout_upper_bound_ms`. Ideally they shouldn't be equal.
 -    `election_timeout_upper_bound_ms` — If the follower does not receive a heartbeat from the leader in this interval, then it must initiate leader election (default: 2000).
 -    `rotate_log_storage_interval` — How many log records to store in a single file (default: 100000).
 -    `reserved_log_items` — How many coordination log records to store before compaction (default: 100000).
--- a/docs/en/operations/server-configuration-parameters/settings.md
+++ b/docs/en/operations/server-configuration-parameters/settings.md
@ -606,6 +606,7 @@ Keys:
 -   `size` – Size of the file. Applies to `log` and `errorlog`. Once the file reaches `size`, ClickHouse archives and renames it, and creates a new log file in its place.
 -   `count` – The number of archived log files that ClickHouse stores.
 -   `console` – Send `log` and `errorlog` to the console instead of file. To enable, set to `1` or `true`.
+-   `stream_compress` – Compress `log` and `errorlog` with `lz4` stream compression. To enable, set to `1` or `true`.

 **Example**

@ -616,6 +617,7 @@ Keys:
    <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
    <size>1000M</size>
    <count>10</count>
+    <stream_compress>true</stream_compress>
 </logger>
 ```

--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@ -3468,7 +3468,7 @@ Possible values:

 Default value: `0`.

-## replication_alter_partitions_sync {#replication-alter-partitions-sync}
+## alter_sync {#alter-sync}

 Allows to set up waiting for actions to be executed on replicas by [ALTER](../../sql-reference/statements/alter/index.md), [OPTIMIZE](../../sql-reference/statements/optimize.md) or [TRUNCATE](../../sql-reference/statements/truncate.md) queries.

--- a/docs/en/sql-reference/functions/arithmetic-functions.md
+++ b/docs/en/sql-reference/functions/arithmetic-functions.md
@ -149,7 +149,7 @@ Compares two values and returns the minimum. The returned value is converted to
 **Syntax**

 ```sql
-max2(value1, value2)
+min2(value1, value2)
 ```

 **Arguments**
--- a/docs/en/sql-reference/functions/json-functions.md
+++ b/docs/en/sql-reference/functions/json-functions.md
@ -4,9 +4,13 @@ sidebar_position: 56
 sidebar_label: JSON
 ---

-# Functions for Working with JSON
+There are two sets of functions to parse JSON. 
+   - `visitParam*` (`simpleJSON*`) is made to parse a special very limited subset of a JSON, but these functions are extremely fast.
+   - `JSONExtract*` is made to parse normal JSON.

-ClickHouse has special functions for working with this JSON. All the JSON functions are based on strong assumptions about what the JSON can be, but they try to do as little as possible to get the job done.
+# visitParam functions
+
+ClickHouse has special functions for working with simplified JSON. All these JSON functions are based on strong assumptions about what the JSON can be, but they try to do as little as possible to get the job done.

 The following assumptions are made:

@ -75,7 +79,9 @@ visitParamExtractString('{"abc":"hello}', 'abc') = '';

 There is currently no support for code points in the format `\uXXXX\uYYYY` that are not from the basic multilingual plane (they are converted to CESU-8 instead of UTF-8).

-The following functions are based on [simdjson](https://github.com/lemire/simdjson) designed for more complex JSON parsing requirements. The assumption 2 mentioned above still applies.
+# JSONExtract functions
+
+The following functions are based on [simdjson](https://github.com/lemire/simdjson) designed for more complex JSON parsing requirements.

 ## isValidJSON(json)

--- a/docs/en/sql-reference/statements/alter/index.md
+++ b/docs/en/sql-reference/statements/alter/index.md
@ -52,12 +52,12 @@ Entries for finished mutations are not deleted right away (the number of preserv

 For non-replicated tables, all `ALTER` queries are performed synchronously. For replicated tables, the query just adds instructions for the appropriate actions to `ZooKeeper`, and the actions themselves are performed as soon as possible. However, the query can wait for these actions to be completed on all the replicas.

-For all `ALTER` queries, you can use the [replication_alter_partitions_sync](/docs/en/operations/settings/settings.md/#replication-alter-partitions-sync) setting to set up waiting.
+For all `ALTER` queries, you can use the [alter_sync](/docs/en/operations/settings/settings.md/#alter-sync) setting to set up waiting.

 You can specify how long (in seconds) to wait for inactive replicas to execute all `ALTER` queries with the [replication_wait_for_inactive_replica_timeout](/docs/en/operations/settings/settings.md/#replication-wait-for-inactive-replica-timeout) setting.

 :::note    
-For all `ALTER` queries, if `replication_alter_partitions_sync = 2` and some replicas are not active for more than the time, specified in the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown.
+For all `ALTER` queries, if `alter_sync = 2` and some replicas are not active for more than the time, specified in the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown.
 :::

 For `ALTER TABLE ... UPDATE|DELETE` queries the synchronicity is defined by the [mutations_sync](/docs/en/operations/settings/settings.md/#mutations_sync) setting.
--- a/docs/en/sql-reference/statements/optimize.md
+++ b/docs/en/sql-reference/statements/optimize.md
@ -19,7 +19,7 @@ OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION I

 The `OPTIMIZE` query is supported for [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) family, the [MaterializedView](../../engines/table-engines/special/materializedview.md) and the [Buffer](../../engines/table-engines/special/buffer.md) engines. Other table engines aren’t supported.

-When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) family of table engines, ClickHouse creates a task for merging and waits for execution on all replicas (if the [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) setting is set to `2`) or on current replica (if the [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) setting is set to `1`).
+When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) family of table engines, ClickHouse creates a task for merging and waits for execution on all replicas (if the [alter_sync](../../operations/settings/settings.md#alter-sync) setting is set to `2`) or on current replica (if the [alter_sync](../../operations/settings/settings.md#alter-sync) setting is set to `1`).

 -   If `OPTIMIZE` does not perform a merge for any reason, it does not notify the client. To enable notifications, use the [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) setting.
 -   If you specify a `PARTITION`, only the specified partition is optimized. [How to set partition expression](alter/partition.md#how-to-set-partition-expression).
@ -29,7 +29,7 @@ When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engin
 You can specify how long (in seconds) to wait for inactive replicas to execute `OPTIMIZE` queries by the [replication_wait_for_inactive_replica_timeout](../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout) setting.

 :::note    
-If the `replication_alter_partitions_sync` is set to `2` and some replicas are not active for more than the time, specified by the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown.
+If the `alter_sync` is set to `2` and some replicas are not active for more than the time, specified by the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown.
 :::

 ## BY expression
--- a/docs/en/sql-reference/statements/truncate.md
+++ b/docs/en/sql-reference/statements/truncate.md
@ -14,10 +14,10 @@ Removes all data from a table. When the clause `IF EXISTS` is omitted, the query

 The `TRUNCATE` query is not supported for [View](../../engines/table-engines/special/view.md), [File](../../engines/table-engines/special/file.md), [URL](../../engines/table-engines/special/url.md), [Buffer](../../engines/table-engines/special/buffer.md) and [Null](../../engines/table-engines/special/null.md) table engines.

-You can use the [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) setting to set up waiting for actions to be executed on replicas.
+You can use the [alter_sync](../../operations/settings/settings.md#alter-sync) setting to set up waiting for actions to be executed on replicas.

 You can specify how long (in seconds) to wait for inactive replicas to execute `TRUNCATE` queries with the [replication_wait_for_inactive_replica_timeout](../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout) setting.

 :::note    
-If the `replication_alter_partitions_sync` is set to `2` and some replicas are not active for more than the time, specified by the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown.
-:::
+If the `alter_sync` is set to `2` and some replicas are not active for more than the time, specified by the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown.
+:::
--- a/docs/ru/operations/settings/settings.md
+++ b/docs/ru/operations/settings/settings.md
@ -3643,7 +3643,7 @@ SETTINGS index_granularity = 8192 │

 Значение по умолчанию: `0`.

-## replication_alter_partitions_sync {#replication-alter-partitions-sync}
+## alter_sync {#alter-sync}

 Позволяет настроить ожидание выполнения действий на репликах запросами [ALTER](../../sql-reference/statements/alter/index.md), [OPTIMIZE](../../sql-reference/statements/optimize.md) или [TRUNCATE](../../sql-reference/statements/truncate.md).

--- a/docs/ru/sql-reference/statements/alter/index.md
+++ b/docs/ru/sql-reference/statements/alter/index.md
@ -72,12 +72,12 @@ ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name

 Для нереплицируемых таблиц, все запросы `ALTER` выполняются синхронно. Для реплицируемых таблиц, запрос всего лишь добавляет инструкцию по соответствующим действиям в `ZooKeeper`, а сами действия осуществляются при первой возможности. Но при этом, запрос может ждать завершения выполнения этих действий на всех репликах.

-Для всех запросов `ALTER` можно настроить ожидание с помощью настройки [replication_alter_partitions_sync](../../../operations/settings/settings.md#replication-alter-partitions-sync).
+Для всех запросов `ALTER` можно настроить ожидание с помощью настройки [alter_sync](../../../operations/settings/settings.md#alter-sync).

 Вы можете указать время ожидания (в секундах) выполнения всех запросов `ALTER` для неактивных реплик с помощью настройки [replication_wait_for_inactive_replica_timeout](../../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout).

 :::info "Примечание"
-    Для всех запросов `ALTER` при `replication_alter_partitions_sync = 2` и неактивности некоторых реплик больше времени, заданного настройкой `replication_wait_for_inactive_replica_timeout`, генерируется исключение `UNFINISHED`.
+    Для всех запросов `ALTER` при `alter_sync = 2` и неактивности некоторых реплик больше времени, заданного настройкой `replication_wait_for_inactive_replica_timeout`, генерируется исключение `UNFINISHED`.
 :::

 Для запросов `ALTER TABLE ... UPDATE|DELETE` синхронность выполнения определяется настройкой [mutations_sync](../../../operations/settings/settings.md#mutations_sync).
--- a/docs/ru/sql-reference/statements/optimize.md
+++ b/docs/ru/sql-reference/statements/optimize.md
@ -19,7 +19,7 @@ OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION I

 Может применяться к таблицам семейства [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md), [MaterializedView](../../engines/table-engines/special/materializedview.md) и [Buffer](../../engines/table-engines/special/buffer.md). Другие движки таблиц не поддерживаются.

-Если запрос `OPTIMIZE` применяется к таблицам семейства [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md), ClickHouse создаёт задачу на слияние и ожидает её исполнения на всех репликах (если значение настройки [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) равно `2`) или на текущей реплике (если значение настройки [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) равно `1`).
+Если запрос `OPTIMIZE` применяется к таблицам семейства [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md), ClickHouse создаёт задачу на слияние и ожидает её исполнения на всех репликах (если значение настройки [alter_sync](../../operations/settings/settings.md#alter-sync) равно `2`) или на текущей реплике (если значение настройки [alter_sync](../../operations/settings/settings.md#alter-sync) равно `1`).

 -   По умолчанию, если запросу `OPTIMIZE` не удалось выполнить слияние, то
 ClickHouse не оповещает клиента. Чтобы включить оповещения, используйте настройку [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop).
@ -30,7 +30,7 @@ ClickHouse не оповещает клиента. Чтобы включить
 Вы можете указать время ожидания (в секундах) выполнения запросов `OPTIMIZE` для неактивных реплик с помощью настройки [replication_wait_for_inactive_replica_timeout](../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout).

 :::info "Примечание"
-    Если значение настройки `replication_alter_partitions_sync` равно `2` и некоторые реплики не активны больше времени, заданного настройкой `replication_wait_for_inactive_replica_timeout`, то генерируется исключение `UNFINISHED`.
+    Если значение настройки `alter_sync` равно `2` и некоторые реплики не активны больше времени, заданного настройкой `replication_wait_for_inactive_replica_timeout`, то генерируется исключение `UNFINISHED`.
 :::

 ## Выражение BY {#by-expression}
--- a/docs/ru/sql-reference/statements/truncate.md
+++ b/docs/ru/sql-reference/statements/truncate.md
@ -14,10 +14,10 @@ TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster]

 Запрос `TRUNCATE` не поддерживается для следующих движков: [View](../../engines/table-engines/special/view.md), [File](../../engines/table-engines/special/file.md), [URL](../../engines/table-engines/special/url.md), [Buffer](../../engines/table-engines/special/buffer.md) и [Null](../../engines/table-engines/special/null.md).

-Вы можете настроить ожидание выполнения действий на репликах с помощью настройки [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync).
+Вы можете настроить ожидание выполнения действий на репликах с помощью настройки [alter_sync](../../operations/settings/settings.md#alter-sync).

 Вы можете указать время ожидания (в секундах) выполнения запросов `TRUNCATE` для неактивных реплик с помощью настройки [replication_wait_for_inactive_replica_timeout](../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout).

 :::info "Примечание"
-    Если значение настройки `replication_alter_partitions_sync` равно `2` и некоторые реплики не активны больше времени, заданного настройкой `replication_wait_for_inactive_replica_timeout`, то генерируется исключение `UNFINISHED`.
+    Если значение настройки `alter_sync` равно `2` и некоторые реплики не активны больше времени, заданного настройкой `replication_wait_for_inactive_replica_timeout`, то генерируется исключение `UNFINISHED`.
 :::
--- a/docs/zh/sql-reference/statements/alter.md
+++ b/docs/zh/sql-reference/statements/alter.md
@ -453,7 +453,7 @@ ALTER TABLE table-name MODIFY TTL ttl-expression

 对于不可复制的表，所有 `ALTER`操作都是同步执行的。对于可复制的表，ALTER操作会将指令添加到ZooKeeper中，然后会尽快的执行它们。然而，该操作可以等待其它所有副本将指令执行完毕。

-对于 `ALTER ... ATTACH|DETACH|DROP`操作，可以通过设置 `replication_alter_partitions_sync` 来启用等待。可用参数值： `0` – 不需要等待; `1` – 仅等待自己执行(默认); `2` – 等待所有节点
+对于 `ALTER ... ATTACH|DETACH|DROP`操作，可以通过设置 `alter_sync` 来启用等待。可用参数值： `0` – 不需要等待; `1` – 仅等待自己执行(默认); `2` – 等待所有节点

 ### Mutations {#alter-mutations}

--- a/docs/zh/sql-reference/statements/alter/index.md
+++ b/docs/zh/sql-reference/statements/alter/index.md
@ -60,7 +60,7 @@ sidebar_label: ALTER

 对于非复制表，所有的 `ALTER` 查询都是同步执行的。对于复制表，查询只是向“ZooKeeper”添加相应动作的指令，动作本身会尽快执行。但是，查询可以等待所有副本上的这些操作完成。

-对于所有的“ALTER”查询，您可以使用[replication_alter_partitions_sync](../../../operations/settings/settings.md#replication-alter-partitions-sync)设置等待。
+对于所有的“ALTER”查询，您可以使用[alter_sync](../../../operations/settings/settings.md#alter-sync)设置等待。

 通过[replication_wait_for_inactive_replica_timeout](../../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout]设置，可以指定不活动的副本执行所有 `ALTER` 查询的等待时间(以秒为单位)。

@ -68,7 +68,7 @@ sidebar_label: ALTER

 !!! info "备注"

-    对于所有的 `ALTER` 查询，如果  `replication_alter_partitions_sync = 2`  和一些副本的不激活时间超过时间(在 `replication_wait_for_inactive_replica_timeout` 设置中指定)，那么将抛出一个异常 `UNFINISHED`。
+    对于所有的 `ALTER` 查询，如果  `alter_sync = 2`  和一些副本的不激活时间超过时间(在 `replication_wait_for_inactive_replica_timeout` 设置中指定)，那么将抛出一个异常 `UNFINISHED`。



--- a/docs/zh/sql-reference/statements/misc.md
+++ b/docs/zh/sql-reference/statements/misc.md
@ -276,7 +276,7 @@ OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION I

 该 `OPTMIZE` 查询也支持 [MaterializedView](../../engines/table-engines/special/materializedview.md) 和 [Buffer](../../engines/table-engines/special/buffer.md) 引擎。 不支持其他表引擎。

-当 `OPTIMIZE` 与 [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) 家族的表引擎一起使用时，ClickHouse将创建一个合并任务，并等待所有节点上的执行（如果 `replication_alter_partitions_sync` 设置已启用）。
+当 `OPTIMIZE` 与 [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) 家族的表引擎一起使用时，ClickHouse将创建一个合并任务，并等待所有节点上的执行（如果 `alter_sync` 设置已启用）。

 -   如果 `OPTIMIZE` 出于任何原因不执行合并，它不通知客户端。 要启用通知，请使用 [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) 设置。
 -   如果您指定 `PARTITION`，仅优化指定的分区。 [如何设置分区表达式](alter.md#alter-how-to-specify-part-expr).
--- a/docs/zh/sql-reference/statements/truncate.md
+++ b/docs/zh/sql-reference/statements/truncate.md
@ -18,7 +18,7 @@ TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster]



-可以使用 replication_alter_partitions_sync 设置在复制集上等待执行的操作。
+可以使用 alter_sync 设置在复制集上等待执行的操作。



@ -27,4 +27,4 @@ TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster]


 !!! info  "注意"
-    如果`replication_alter_partitions_sync` 被设置为`2`，并且某些复制集超过 `replication_wait_for_inactive_replica_timeout`设置的时间不激活，那么将抛出一个异常`UNFINISHED`。
+    如果`alter_sync` 被设置为`2`，并且某些复制集超过 `replication_wait_for_inactive_replica_timeout`设置的时间不激活，那么将抛出一个异常`UNFINISHED`。
--- a/programs/copier/ClusterCopier.cpp
+++ b/programs/copier/ClusterCopier.cpp
@ -622,7 +622,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t
        Settings settings_push = task_cluster->settings_push;
        ClusterExecutionMode execution_mode = ClusterExecutionMode::ON_EACH_NODE;

-        if (settings_push.replication_alter_partitions_sync == 1)
+        if (settings_push.alter_sync == 1)
            execution_mode = ClusterExecutionMode::ON_EACH_SHARD;

        query_alter_ast_string += " ALTER TABLE " + getQuotedTable(original_table) +
@ -640,7 +640,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t
                task_cluster->settings_push,
                execution_mode);

-            if (settings_push.replication_alter_partitions_sync == 1)
+            if (settings_push.alter_sync == 1)
            {
                LOG_INFO(
                    log,
@ -863,7 +863,7 @@ bool ClusterCopier::tryDropPartitionPiece(
        Settings settings_push = task_cluster->settings_push;

        /// It is important, DROP PARTITION must be done synchronously
-        settings_push.replication_alter_partitions_sync = 2;
+        settings_push.alter_sync = 2;

        LOG_INFO(log, "Execute distributed DROP PARTITION: {}", query);
        /// We have to drop partition_piece on each replica
--- a/programs/copier/TaskCluster.cpp
+++ b/programs/copier/TaskCluster.cpp
@ -67,7 +67,7 @@ void DB::TaskCluster::reloadSettings(const Poco::Util::AbstractConfiguration & c
    set_default_value(settings_pull.preferred_block_size_bytes, 0);

    set_default_value(settings_push.insert_distributed_timeout, 0);
-    set_default_value(settings_push.replication_alter_partitions_sync, 2);
+    set_default_value(settings_push.alter_sync, 2);
 }

 }
--- a/programs/format/Format.cpp
+++ b/programs/format/Format.cpp
@ -63,8 +63,9 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
        Settings cmd_settings;
        for (const auto & field : cmd_settings.all())
        {
-            if (field.getName() == "max_parser_depth" || field.getName() == "max_query_size")
-                cmd_settings.addProgramOption(desc, field);
+            std::string_view name = field.getName();
+            if (name == "max_parser_depth" || name == "max_query_size")
+                cmd_settings.addProgramOption(desc, name, field);
        }

        boost::program_options::variables_map options;
--- a/rust/CMakeLists.txt
+++ b/rust/CMakeLists.txt
@ -51,8 +51,20 @@ file(COPY ".cargo" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
 function(add_rust_subdirectory src)
    set(dst "${CMAKE_CURRENT_BINARY_DIR}/${src}")
    message(STATUS "Copy ${src} to ${dst}")
-    file(COPY "${src}" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
+    file(COPY "${src}" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}"
+        PATTERN target EXCLUDE)
    add_subdirectory("${dst}" "${dst}")
+
+    # cmake -E copy* do now know how to exclude files
+    # but we need to exclude "target" folder from copying, if someone or semantic
+    # completion created it.
+    add_custom_target(${src}-update ALL
+        COMMAND ${CMAKE_COMMAND}
+            -DFROM=${src}
+            -DTO=${CMAKE_CURRENT_BINARY_DIR}
+            -P "${CMAKE_CURRENT_SOURCE_DIR}/copy_exclude.cmake"
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        VERBATIM)
 endfunction()

 add_rust_subdirectory (BLAKE3)
--- a/rust/copy_exclude.cmake
+++ b/rust/copy_exclude.cmake
@ -0,0 +1,2 @@
+file(COPY "${FROM}" DESTINATION "${TO}"
+    PATTERN target EXCLUDE)
--- a/src/Access/SettingsConstraints.cpp
+++ b/src/Access/SettingsConstraints.cpp
@ -34,15 +34,22 @@ SettingsConstraints::~SettingsConstraints() = default;
 void SettingsConstraints::clear()
 {
    constraints.clear();
+    settings_alias_cache.clear();
 }

 void SettingsConstraints::set(const String & full_name, const Field & min_value, const Field & max_value, SettingConstraintWritability writability)
 {
-    auto & constraint = constraints[full_name];
+    std::string resolved_name{resolveSettingName(full_name)};
+
+    auto & constraint = constraints[resolved_name];
+
+    if (full_name != resolved_name)
+        settings_alias_cache[full_name] = resolved_name;
+
    if (!min_value.isNull())
-        constraint.min_value = settingCastValueUtil(full_name, min_value);
+        constraint.min_value = settingCastValueUtil(resolved_name, min_value);
    if (!max_value.isNull())
-        constraint.max_value = settingCastValueUtil(full_name, max_value);
+        constraint.max_value = settingCastValueUtil(resolved_name, max_value);
    constraint.writability = writability;
 }

@ -68,7 +75,9 @@ void SettingsConstraints::merge(const SettingsConstraints & other)
    if (access_control->doesSettingsConstraintsReplacePrevious())
    {
        for (const auto & [other_name, other_constraint] : other.constraints)
+        {
            constraints[other_name] = other_constraint;
+        }
    }
    else
    {
@ -83,6 +92,9 @@ void SettingsConstraints::merge(const SettingsConstraints & other)
                constraint.writability = SettingConstraintWritability::CONST; // NOTE: In this mode <readonly/> flag cannot be overridden to be false
        }
    }
+
+    for (const auto & [other_alias, other_resolved_name] : settings_alias_cache)
+        settings_alias_cache.try_emplace(other_alias, other_resolved_name);
 }


@ -117,14 +129,15 @@ void SettingsConstraints::check(const Settings & current_settings, const Setting
        if (element.writability)
            new_value = *element.writability;

-        auto it = constraints.find(element.setting_name);
+        auto setting_name = Settings::Traits::resolveName(element.setting_name);
+        auto it = constraints.find(setting_name);
        if (it != constraints.end())
            old_value = it->second.writability;

        if (new_value != old_value)
        {
            if (old_value == SettingConstraintWritability::CONST)
-                throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "Setting {} should not be changed", element.setting_name);
+                throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "Setting {} should not be changed", setting_name);
        }
    }
 }
@ -204,7 +217,7 @@ bool getNewValueToCheck(const T & current_settings, SettingChange & change, Fiel

 bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const
 {
-    const String & setting_name = change.name;
+    std::string_view setting_name = Settings::Traits::resolveName(change.name);

    if (setting_name == "profile")
        return true;
@ -247,7 +260,15 @@ bool SettingsConstraints::checkImpl(const MergeTreeSettings & current_settings,

 bool SettingsConstraints::Checker::check(SettingChange & change, const Field & new_value, ReactionOnViolation reaction) const
 {
-    const String & setting_name = change.name;
+    if (!explain.empty())
+    {
+        if (reaction == THROW_ON_VIOLATION)
+            throw Exception::createDeprecated(explain, code);
+        else
+            return false;
+    }
+
+    std::string_view setting_name = setting_name_resolver(change.name);

    auto less_or_cannot_compare = [=](const Field & left, const Field & right)
    {
@ -263,13 +284,6 @@ bool SettingsConstraints::Checker::check(SettingChange & change, const Field & n
        }
    };

-    if (!explain.empty())
-    {
-        if (reaction == THROW_ON_VIOLATION)
-            throw Exception::createDeprecated(explain, code);
-        else
-            return false;
-    }

    if (constraint.writability == SettingConstraintWritability::CONST)
    {
@ -315,9 +329,17 @@ bool SettingsConstraints::Checker::check(SettingChange & change, const Field & n
    return true;
 }

+std::string_view SettingsConstraints::resolveSettingNameWithCache(std::string_view name) const
+{
+    if (auto it = settings_alias_cache.find(name); it != settings_alias_cache.end())
+        return it->second;
+    return name;
+}
+
 SettingsConstraints::Checker SettingsConstraints::getChecker(const Settings & current_settings, std::string_view setting_name) const
 {
-    if (!current_settings.allow_ddl && setting_name == "allow_ddl")
+    auto resolved_name = resolveSettingNameWithCache(setting_name);
+    if (!current_settings.allow_ddl && resolved_name == "allow_ddl")
        return Checker("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED);

    /** The `readonly` value is understood as follows:
@ -326,10 +348,10 @@ SettingsConstraints::Checker SettingsConstraints::getChecker(const Settings & cu
      * 2 - only read requests, as well as changing settings, except for the `readonly` setting.
      */

-    if (current_settings.readonly > 1 && setting_name == "readonly")
+    if (current_settings.readonly > 1 && resolved_name == "readonly")
        return Checker("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY);

-    auto it = constraints.find(setting_name);
+    auto it = constraints.find(resolved_name);
    if (current_settings.readonly == 1)
    {
        if (it == constraints.end() || it->second.writability != SettingConstraintWritability::CHANGEABLE_IN_READONLY)
@ -338,17 +360,18 @@ SettingsConstraints::Checker SettingsConstraints::getChecker(const Settings & cu
    else // For both readonly=0 and readonly=2
    {
        if (it == constraints.end())
-            return Checker(); // Allowed
+            return Checker(Settings::Traits::resolveName); // Allowed
    }
-    return Checker(it->second);
+    return Checker(it->second, Settings::Traits::resolveName);
 }

 SettingsConstraints::Checker SettingsConstraints::getMergeTreeChecker(std::string_view short_name) const
 {
-    auto it = constraints.find(settingFullName<MergeTreeSettings>(short_name));
+    auto full_name = settingFullName<MergeTreeSettings>(short_name);
+    auto it = constraints.find(resolveSettingNameWithCache(full_name));
    if (it == constraints.end())
-        return Checker(); // Allowed
-    return Checker(it->second);
+        return Checker(MergeTreeSettings::Traits::resolveName); // Allowed
+    return Checker(it->second, MergeTreeSettings::Traits::resolveName);
 }

 bool SettingsConstraints::Constraint::operator==(const Constraint & other) const
--- a/src/Access/SettingsConstraints.h
+++ b/src/Access/SettingsConstraints.h
@ -108,11 +108,16 @@ private:
    struct Checker
    {
        Constraint constraint;
+        using NameResolver = std::function<std::string_view(std::string_view)>;
+        NameResolver setting_name_resolver;
+
        String explain;
        int code = 0;

        // Allows everything
-        Checker() = default;
+        explicit Checker(NameResolver setting_name_resolver_)
+            : setting_name_resolver(std::move(setting_name_resolver_))
+        {}

        // Forbidden with explanation
        Checker(const String & explain_, int code_)
@ -122,8 +127,9 @@ private:
        {}

        // Allow or forbid depending on range defined by constraint, also used to return stored constraint
-        explicit Checker(const Constraint & constraint_)
+        explicit Checker(const Constraint & constraint_, NameResolver setting_name_resolver_)
            : constraint(constraint_)
+            , setting_name_resolver(std::move(setting_name_resolver_))
        {}

        // Perform checking
@ -137,10 +143,6 @@ private:
        {
            return std::hash<std::string_view>{}(txt);
        }
-        size_t operator()(const String & txt) const
-        {
-            return std::hash<String>{}(txt);
-        }
    };

    bool checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const;
@ -149,9 +151,15 @@ private:
    Checker getChecker(const Settings & current_settings, std::string_view setting_name) const;
    Checker getMergeTreeChecker(std::string_view short_name) const;

+    std::string_view resolveSettingNameWithCache(std::string_view name) const;
+
    // Special container for heterogeneous lookups: to avoid `String` construction during `find(std::string_view)`
    using Constraints = std::unordered_map<String, Constraint, StringHash, std::equal_to<>>;
    Constraints constraints;
+    /// to avoid creating new string every time we cache the alias resolution
+    /// we cannot use resolveName from BaseSettings::Traits because MergeTreeSettings have added prefix
+    /// we store only resolved aliases inside the Constraints so to correctly search the container we always need to use resolved name
+    std::unordered_map<std::string, std::string, StringHash, std::equal_to<>> settings_alias_cache;

    const AccessControl * access_control;
 };
--- a/src/Access/SettingsProfileElement.cpp
+++ b/src/Access/SettingsProfileElement.cpp
@ -14,11 +14,6 @@
 namespace DB
 {

-namespace
-{
-    constexpr const char ALLOW_BACKUP_SETTING_NAME[] = "allow_backup";
-}
-
 namespace ErrorCodes
 {
    extern const int NOT_IMPLEMENTED;
@ -54,7 +49,7 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A
        if (access_control)
        {
            /// Check if a setting with that name is allowed.
-            if (setting_name != ALLOW_BACKUP_SETTING_NAME)
+            if (!SettingsProfileElements::isAllowBackupSetting(setting_name))
                access_control->checkSettingNameIsAllowed(setting_name);
            /// Check if a CHANGEABLE_IN_READONLY is allowed.
            if (ast.writability == SettingConstraintWritability::CHANGEABLE_IN_READONLY && !access_control->doesSettingsConstraintsReplacePrevious())
@ -192,11 +187,8 @@ Settings SettingsProfileElements::toSettings() const
    Settings res;
    for (const auto & elem : *this)
    {
-        if (!elem.setting_name.empty() && (elem.setting_name != ALLOW_BACKUP_SETTING_NAME))
-        {
-            if (!elem.value.isNull())
-                res.set(elem.setting_name, elem.value);
-        }
+        if (!elem.setting_name.empty() && !isAllowBackupSetting(elem.setting_name) && !elem.value.isNull())
+            res.set(elem.setting_name, elem.value);
    }
    return res;
 }
@ -206,7 +198,7 @@ SettingsChanges SettingsProfileElements::toSettingsChanges() const
    SettingsChanges res;
    for (const auto & elem : *this)
    {
-        if (!elem.setting_name.empty() && (elem.setting_name != ALLOW_BACKUP_SETTING_NAME))
+        if (!elem.setting_name.empty() && !isAllowBackupSetting(elem.setting_name))
        {
            if (!elem.value.isNull())
                res.push_back({elem.setting_name, elem.value});
@ -219,7 +211,7 @@ SettingsConstraints SettingsProfileElements::toSettingsConstraints(const AccessC
 {
    SettingsConstraints res{access_control};
    for (const auto & elem : *this)
-        if (!elem.setting_name.empty() && elem.isConstraint() && elem.setting_name != ALLOW_BACKUP_SETTING_NAME)
+        if (!elem.setting_name.empty() && elem.isConstraint() && !isAllowBackupSetting(elem.setting_name))
            res.set(
                elem.setting_name,
                elem.min_value,
@ -248,7 +240,7 @@ bool SettingsProfileElements::isBackupAllowed() const
 {
    for (const auto & setting : *this)
    {
-        if (setting.setting_name == ALLOW_BACKUP_SETTING_NAME)
+        if (isAllowBackupSetting(setting.setting_name))
            return static_cast<bool>(SettingFieldBool{setting.value});
    }
    return true;
@ -256,7 +248,8 @@ bool SettingsProfileElements::isBackupAllowed() const

 bool SettingsProfileElements::isAllowBackupSetting(const String & setting_name)
 {
-    return setting_name == ALLOW_BACKUP_SETTING_NAME;
+    static constexpr std::string_view ALLOW_BACKUP_SETTING_NAME = "allow_backup";
+    return Settings::Traits::resolveName(setting_name) == ALLOW_BACKUP_SETTING_NAME;
 }

 }
--- a/src/Access/resolveSetting.h
+++ b/src/Access/resolveSetting.h
@ -87,4 +87,12 @@ inline String settingFullName<MergeTreeSettings>(std::string_view short_name)
    return full_name;
 }

+inline std::string resolveSettingName(std::string_view full_name)
+{
+    return resolveSetting(full_name, [&] <typename T> (std::string_view short_name, SettingsType<T>)
+    {
+        return settingFullName<T>(T::Traits::resolveName(short_name));
+    });
+}
+
 }
--- a/src/Analyzer/Passes/ArrayExistsToHasPass.cpp
+++ b/src/Analyzer/Passes/ArrayExistsToHasPass.cpp
@ -0,0 +1,85 @@
+#include <Functions/FunctionFactory.h>
+
+#include <Interpreters/Context.h>
+
+#include <Analyzer/ColumnNode.h>
+#include <Analyzer/ConstantNode.h>
+#include <Analyzer/FunctionNode.h>
+#include <Analyzer/InDepthQueryTreeVisitor.h>
+#include <Analyzer/LambdaNode.h>
+
+#include "ArrayExistsToHasPass.h"
+
+namespace DB
+{
+namespace
+{
+    class RewriteArrayExistsToHasVisitor : public InDepthQueryTreeVisitorWithContext<RewriteArrayExistsToHasVisitor>
+    {
+    public:
+        using Base = InDepthQueryTreeVisitorWithContext<RewriteArrayExistsToHasVisitor>;
+        using Base::Base;
+
+        void visitImpl(QueryTreeNodePtr & node)
+        {
+            if (!getSettings().optimize_rewrite_array_exists_to_has)
+                return;
+
+            auto * function_node = node->as<FunctionNode>();
+            if (!function_node || function_node->getFunctionName() != "arrayExists")
+                return;
+
+            auto & function_arguments_nodes = function_node->getArguments().getNodes();
+            if (function_arguments_nodes.size() != 2)
+                return;
+
+            /// lambda function must be like: x -> x = elem
+            auto * lambda_node = function_arguments_nodes[0]->as<LambdaNode>();
+            if (!lambda_node)
+                return;
+
+            auto & lambda_arguments_nodes = lambda_node->getArguments().getNodes();
+            if (lambda_arguments_nodes.size() != 1)
+                return;
+            auto * column_node = lambda_arguments_nodes[0]->as<ColumnNode>();
+
+            auto * filter_node = lambda_node->getExpression()->as<FunctionNode>();
+            if (!filter_node || filter_node->getFunctionName() != "equals")
+                return;
+
+            auto filter_arguments_nodes = filter_node->getArguments().getNodes();
+            if (filter_arguments_nodes.size() != 2)
+                return;
+
+            ColumnNode * filter_column_node = nullptr;
+            if (filter_arguments_nodes[1]->as<ConstantNode>() && (filter_column_node = filter_arguments_nodes[0]->as<ColumnNode>())
+                && filter_column_node->getColumnName() == column_node->getColumnName())
+            {
+                /// Rewrite arrayExists(x -> x = elem, arr) -> has(arr, elem)
+                function_arguments_nodes[0] = std::move(function_arguments_nodes[1]);
+                function_arguments_nodes[1] = std::move(filter_arguments_nodes[1]);
+                function_node->resolveAsFunction(
+                    FunctionFactory::instance().get("has", getContext())->build(function_node->getArgumentColumns()));
+            }
+            else if (
+                filter_arguments_nodes[0]->as<ConstantNode>() && (filter_column_node = filter_arguments_nodes[1]->as<ColumnNode>())
+                && filter_column_node->getColumnName() == column_node->getColumnName())
+            {
+                /// Rewrite arrayExists(x -> elem = x, arr) -> has(arr, elem)
+                function_arguments_nodes[0] = std::move(function_arguments_nodes[1]);
+                function_arguments_nodes[1] = std::move(filter_arguments_nodes[0]);
+                function_node->resolveAsFunction(
+                    FunctionFactory::instance().get("has", getContext())->build(function_node->getArgumentColumns()));
+            }
+        }
+    };
+
+}
+
+void RewriteArrayExistsToHasPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
+{
+    RewriteArrayExistsToHasVisitor visitor(context);
+    visitor.visit(query_tree_node);
+}
+
+}
--- a/src/Analyzer/Passes/ArrayExistsToHasPass.h
+++ b/src/Analyzer/Passes/ArrayExistsToHasPass.h
@ -0,0 +1,18 @@
+#pragma once
+
+#include <Analyzer/IQueryTreePass.h>
+
+namespace DB
+{
+/// Rewrite possible 'arrayExists(func, arr)' to 'has(arr, elem)' to improve performance
+/// arrayExists(x -> x = 1, arr) -> has(arr, 1)
+class RewriteArrayExistsToHasPass final : public IQueryTreePass
+{
+public:
+    String getName() override { return "RewriteArrayExistsToHas"; }
+
+    String getDescription() override { return "Rewrite arrayExists(func, arr) functions to has(arr, elem) when logically equivalent"; }
+
+    void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
+};
+}
--- a/src/Analyzer/QueryTreePassManager.cpp
+++ b/src/Analyzer/QueryTreePassManager.cpp
@ -35,6 +35,7 @@
 #include <Analyzer/Passes/ConvertOrLikeChainPass.h>
 #include <Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h>
 #include <Analyzer/Passes/GroupingFunctionsResolvePass.h>
+#include <Analyzer/Passes/ArrayExistsToHasPass.h>

 namespace DB
 {
@ -217,6 +218,7 @@ void addQueryTreePasses(QueryTreePassManager & manager)
    manager.addPass(std::make_unique<CountDistinctPass>());
    manager.addPass(std::make_unique<RewriteAggregateFunctionWithIfPass>());
    manager.addPass(std::make_unique<SumIfToCountIfPass>());
+    manager.addPass(std::make_unique<RewriteArrayExistsToHasPass>());
    manager.addPass(std::make_unique<NormalizeCountVariantsPass>());

    manager.addPass(std::make_unique<CustomizeFunctionsPass>());
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@ -2359,6 +2359,22 @@ void ClientBase::showClientVersion()
    std::cout << DBMS_NAME << " " + getName() + " version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
 }

+namespace
+{
+
+/// Define transparent hash to we can use
+/// std::string_view with the containers
+struct TransparentStringHash
+{
+    using is_transparent = void;
+    size_t operator()(std::string_view txt) const
+    {
+        return std::hash<std::string_view>{}(txt);
+    }
+};
+
+}
+

 void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments)
 {
@ -2374,19 +2390,35 @@ void ClientBase::parseAndCheckOptions(OptionsDescription & options_description,
        /// skip ambiguous merge tree settings.
        auto & main_options = options_description.main_description.value();

-        NameSet main_option_names;
+        std::unordered_set<std::string, TransparentStringHash, std::equal_to<>> main_option_names;
        for (const auto & option : main_options.options())
            main_option_names.insert(option->long_name());

        for (const auto & setting : cmd_merge_tree_settings.all())
        {
-            if (main_option_names.contains(setting.getName()))
-                continue;
+            const auto add_setting = [&](const std::string_view name)
+            {
+                if (auto it = main_option_names.find(name); it != main_option_names.end())
+                    return;

-            if (allow_repeated_settings)
-                cmd_merge_tree_settings.addProgramOptionAsMultitoken(main_options, setting);
-            else
-                cmd_merge_tree_settings.addProgramOption(main_options, setting);
+                if (allow_repeated_settings)
+                    cmd_merge_tree_settings.addProgramOptionAsMultitoken(main_options, name, setting);
+                else
+                    cmd_merge_tree_settings.addProgramOption(main_options, name, setting);
+            };
+
+            const auto & setting_name = setting.getName();
+
+            add_setting(setting_name);
+
+            const auto & settings_to_aliases = MergeTreeSettings::Traits::settingsToAliases();
+            if (auto it = settings_to_aliases.find(setting_name); it != settings_to_aliases.end())
+            {
+                for (const auto alias : it->second)
+                {
+                    add_setting(alias);
+                }
+            }
        }
    }

--- a/src/Common/CurrentMetrics.cpp
+++ b/src/Common/CurrentMetrics.cpp
@ -110,6 +110,9 @@
    M(ThreadsInOvercommitTracker, "Number of waiting threads inside of OvercommitTracker") \
    M(IOUringPendingEvents, "Number of io_uring SQEs waiting to be submitted") \
    M(IOUringInFlightEvents, "Number of io_uring SQEs in flight") \
+    M(ReadTaskRequestsSent, "The current number of callback requests in flight from the remote server back to the initiator server to choose the read task (for s3Cluster table function and similar). Measured on the remote server side.") \
+    M(MergeTreeReadTaskRequestsSent, "The current number of callback requests in flight from the remote server back to the initiator server to choose the read task (for MergeTree tables). Measured on the remote server side.") \
+    M(MergeTreeAllRangesAnnouncementsSent, "The current number of announcement being sent in flight from the remote server to the initiator server about the set of data parts (for MergeTree tables). Measured on the remote server side.")

 namespace CurrentMetrics
 {
--- a/src/Common/HashTable/Hash.h
+++ b/src/Common/HashTable/Hash.h
@ -94,7 +94,7 @@ inline DB::UInt64 intHashCRC32(DB::UInt64 x)
 #elif (defined(__PPC64__) || defined(__powerpc64__)) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
    return crc32_ppc(-1U, reinterpret_cast<const unsigned char *>(&x), sizeof(x));
 #elif defined(__s390x__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-    return s390x_crc32(-1U, x)
+    return s390x_crc32(-1U, x);
 #else
    /// On other platforms we do not have CRC32. NOTE This can be confusing.
    /// NOTE: consider using intHash32()
--- a/src/Common/HashTable/HashTable.h
+++ b/src/Common/HashTable/HashTable.h
@ -41,6 +41,7 @@ namespace ErrorCodes
 {
    extern const int LOGICAL_ERROR;
    extern const int NO_AVAILABLE_DATA;
+    extern const int CANNOT_ALLOCATE_MEMORY;
 }
 }

@ -507,9 +508,21 @@ protected:
        return place_value;
    }

+    static size_t allocCheckOverflow(size_t buffer_size)
+    {
+        size_t size = 0;
+        if (common::mulOverflow(buffer_size, sizeof(Cell), size))
+            throw DB::Exception(
+                DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY,
+                "Integer overflow trying to allocate memory for HashTable. Trying to allocate {} cells of {} bytes each",
+                buffer_size, sizeof(Cell));
+
+        return size;
+    }
+
    void alloc(const Grower & new_grower)
    {
-        buf = reinterpret_cast<Cell *>(Allocator::alloc(new_grower.bufSize() * sizeof(Cell)));
+        buf = reinterpret_cast<Cell *>(Allocator::alloc(allocCheckOverflow(new_grower.bufSize())));
        grower = new_grower;
    }

@ -566,11 +579,11 @@ protected:

        if constexpr (Cell::need_to_notify_cell_during_move)
        {
-            buf = reinterpret_cast<Cell *>(Allocator::alloc(new_grower.bufSize() * sizeof(Cell)));
+            buf = reinterpret_cast<Cell *>(Allocator::alloc(allocCheckOverflow(new_grower.bufSize())));
            memcpy(reinterpret_cast<void *>(buf), reinterpret_cast<const void *>(old_buffer.get()), old_buffer_size);
        }
        else
-            buf = reinterpret_cast<Cell *>(Allocator::realloc(buf, old_buffer_size, new_grower.bufSize() * sizeof(Cell)));
+            buf = reinterpret_cast<Cell *>(Allocator::realloc(buf, old_buffer_size, allocCheckOverflow(new_grower.bufSize())));

        grower = new_grower;

--- a/src/Common/ProfileEvents.cpp
+++ b/src/Common/ProfileEvents.cpp
@ -479,6 +479,16 @@ The server successfully detected this situation and will download merged part fr
    M(IOUringSQEsResubmits, "Total number of io_uring SQE resubmits performed") \
    M(IOUringCQEsCompleted, "Total number of successfully completed io_uring CQEs") \
    M(IOUringCQEsFailed, "Total number of completed io_uring CQEs with failures") \
+    \
+    M(ReadTaskRequestsReceived, "The number of callbacks requested from the remote server back to the initiator server to choose the read task (for s3Cluster table function and similar). Measured on the initiator server side.") \
+    M(MergeTreeReadTaskRequestsReceived, "The number of callbacks requested from the remote server back to the initiator server to choose the read task (for MergeTree tables). Measured on the initiator server side.") \
+    \
+    M(ReadTaskRequestsSent, "The number of callbacks requested from the remote server back to the initiator server to choose the read task (for s3Cluster table function and similar). Measured on the remote server side.") \
+    M(MergeTreeReadTaskRequestsSent, "The number of callbacks requested from the remote server back to the initiator server to choose the read task (for MergeTree tables). Measured on the remote server side.") \
+    M(MergeTreeAllRangesAnnouncementsSent, "The number of announcement sent from the remote server to the initiator server about the set of data parts (for MergeTree tables). Measured on the remote server side.") \
+    M(ReadTaskRequestsSentElapsedMicroseconds, "Time spent in callbacks requested from the remote server back to the initiator server to choose the read task (for s3Cluster table function and similar). Measured on the remote server side.") \
+    M(MergeTreeReadTaskRequestsSentElapsedMicroseconds, "Time spent in callbacks requested from the remote server back to the initiator server to choose the read task (for MergeTree tables). Measured on the remote server side.") \
+    M(MergeTreeAllRangesAnnouncementsSentElapsedMicroseconds, "Time spent in sending the announcement from the remote server to the initiator server about the set of data parts (for MergeTree tables). Measured on the remote server side.")

 namespace ProfileEvents
 {
--- a/src/Common/StackTrace.cpp
+++ b/src/Common/StackTrace.cpp
@ -221,6 +221,8 @@ static void * getCallerAddress(const ucontext_t & context)
    return reinterpret_cast<void *>(context.uc_mcontext.mc_srr0);
 #elif defined(__riscv)
    return reinterpret_cast<void *>(context.uc_mcontext.__gregs[REG_PC]);
+#elif defined(__s390x__)
+    return reinterpret_cast<void *>(context.uc_mcontext.psw.addr);
 #else
    return nullptr;
 #endif
--- a/src/Common/ThreadFuzzer.cpp
+++ b/src/Common/ThreadFuzzer.cpp
@ -338,6 +338,8 @@ void ThreadFuzzer::setup() const
    #    define GLIBC_SYMVER "GLIBC_2.27"
    #elif (defined(__PPC64__) || defined(__powerpc64__)) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
    #    define GLIBC_SYMVER "GLIBC_2.17"
+    #elif (defined(__S390X__) || defined(__s390x__))
+    #    define GLIBC_SYMVER "GLIBC_2.2"
    #else
    #    error Your platform is not supported.
    #endif
--- a/src/Common/formatIPv6.cpp
+++ b/src/Common/formatIPv6.cpp
@ -165,9 +165,9 @@ void formatIPv6(const unsigned char * src, char *& dst, uint8_t zeroed_tail_byte
            uint8_t ipv4_buffer[IPV4_BINARY_LENGTH] = {0};
            memcpy(ipv4_buffer, src + 12, IPV4_BINARY_LENGTH);
            // Due to historical reasons formatIPv4() takes ipv4 in BE format, but inside ipv6 we store it in LE-format.
-            if constexpr (std::endian::native == std::endian::little)
-                std::reverse(std::begin(ipv4_buffer), std::end(ipv4_buffer));
-
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+            std::reverse(std::begin(ipv4_buffer), std::end(ipv4_buffer));
+#endif
            formatIPv4(ipv4_buffer, dst, std::min(zeroed_tail_bytes_count, static_cast<uint8_t>(IPV4_BINARY_LENGTH)), "0");
            // formatIPv4 has already added a null-terminator for us.
            return;
--- a/src/Common/waitForPid.cpp
+++ b/src/Common/waitForPid.cpp
@ -45,6 +45,8 @@ enum PollPidResult
        #define SYS_pidfd_open 434
    #elif defined(__riscv)
        #define SYS_pidfd_open 434
+    #elif defined(__s390x__)
+        #define SYS_pidfd_open 434
    #else
        #error "Unsupported architecture"
    #endif
--- a/src/Compression/CompressionFactory.cpp
+++ b/src/Compression/CompressionFactory.cpp
@ -195,9 +195,9 @@ CompressionCodecFactory::CompressionCodecFactory()
    registerCodecGorilla(*this);
    registerCodecEncrypted(*this);
    registerCodecFPC(*this);
-    #ifdef ENABLE_QPL_COMPRESSION
-        registerCodecDeflateQpl(*this);
-    #endif
+#ifdef ENABLE_QPL_COMPRESSION
+    registerCodecDeflateQpl(*this);
+#endif
 #endif

    default_codec = get("LZ4", {});
--- a/src/Compression/LZ4_decompress_faster.cpp
+++ b/src/Compression/LZ4_decompress_faster.cpp
@ -81,7 +81,7 @@ inline void copyOverlap8(UInt8 * op, const UInt8 *& match, size_t offset)
 }


-#if defined(__x86_64__) || defined(__PPC__) || defined(__riscv)
+#if defined(__x86_64__) || defined(__PPC__) || defined(__s390x__) || defined(__riscv)

 /** We use 'xmm' (128bit SSE) registers here to shuffle 16 bytes.
  *
@ -272,7 +272,7 @@ inline void copyOverlap16(UInt8 * op, const UInt8 *& match, const size_t offset)
 }


-#if defined(__x86_64__) || defined(__PPC__) || defined (__riscv)
+#if defined(__x86_64__) || defined(__PPC__) || defined(__s390x__) || defined (__riscv)

 inline void copyOverlap16Shuffle(UInt8 * op, const UInt8 *& match, const size_t offset)
 {
--- a/src/Coordination/CoordinationSettings.h
+++ b/src/Coordination/CoordinationSettings.h
@ -18,7 +18,7 @@ struct Settings;
  */


-#define LIST_OF_COORDINATION_SETTINGS(M) \
+#define LIST_OF_COORDINATION_SETTINGS(M, ALIAS) \
    M(Milliseconds, min_session_timeout_ms, Coordination::DEFAULT_MIN_SESSION_TIMEOUT_MS, "Min client session timeout", 0) \
    M(Milliseconds, session_timeout_ms, Coordination::DEFAULT_MAX_SESSION_TIMEOUT_MS, "Max client session timeout", 0) \
    M(Milliseconds, operation_timeout_ms, Coordination::DEFAULT_OPERATION_TIMEOUT_MS, "Default client operation timeout", 0) \
--- a/src/Coordination/KeeperServer.cpp
+++ b/src/Coordination/KeeperServer.cpp
@ -273,6 +273,19 @@ void KeeperServer::launchRaftServer(const Poco::Util::AbstractConfiguration & co
        coordination_settings->election_timeout_lower_bound_ms.totalMilliseconds(), "election_timeout_lower_bound_ms", log);
    params.election_timeout_upper_bound_ = getValueOrMaxInt32AndLogWarning(
        coordination_settings->election_timeout_upper_bound_ms.totalMilliseconds(), "election_timeout_upper_bound_ms", log);
+
+    if (params.election_timeout_lower_bound_ || params.election_timeout_upper_bound_)
+    {
+        if (params.election_timeout_lower_bound_ >= params.election_timeout_upper_bound_)
+        {
+            LOG_FATAL(
+                log,
+                "election_timeout_lower_bound_ms is greater than election_timeout_upper_bound_ms, this would disable leader election "
+                "completely.");
+            std::terminate();
+        }
+    }
+
    params.reserved_log_items_ = getValueOrMaxInt32AndLogWarning(coordination_settings->reserved_log_items, "reserved_log_items", log);
    params.snapshot_distance_ = getValueOrMaxInt32AndLogWarning(coordination_settings->snapshot_distance, "snapshot_distance", log);

--- a/src/Core/BaseSettings.h
+++ b/src/Core/BaseSettings.h
@ -134,8 +134,8 @@ public:
    /// (Don't forget to call notify() on the `variables_map` after parsing it!)
    void addProgramOptionsAsMultitokens(boost::program_options::options_description & options);

-    void addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field);
-    void addProgramOptionAsMultitoken(boost::program_options::options_description & options, const SettingFieldRef & field);
+    void addProgramOption(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field);
+    void addProgramOptionAsMultitoken(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field);

    enum SkipFlags
    {
@ -219,6 +219,7 @@ struct BaseSettingsHelpers
 template <typename TTraits>
 void BaseSettings<TTraits>::set(std::string_view name, const Field & value)
 {
+    name = TTraits::resolveName(name);
    const auto & accessor = Traits::Accessor::instance();
    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
        accessor.setValue(*this, index, value);
@ -229,6 +230,7 @@ void BaseSettings<TTraits>::set(std::string_view name, const Field & value)
 template <typename TTraits>
 Field BaseSettings<TTraits>::get(std::string_view name) const
 {
+    name = TTraits::resolveName(name);
    const auto & accessor = Traits::Accessor::instance();
    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
        return accessor.getValue(*this, index);
@ -239,6 +241,7 @@ Field BaseSettings<TTraits>::get(std::string_view name) const
 template <typename TTraits>
 void BaseSettings<TTraits>::setString(std::string_view name, const String & value)
 {
+    name = TTraits::resolveName(name);
    const auto & accessor = Traits::Accessor::instance();
    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
        accessor.setValueString(*this, index, value);
@ -249,6 +252,7 @@ void BaseSettings<TTraits>::setString(std::string_view name, const String & valu
 template <typename TTraits>
 String BaseSettings<TTraits>::getString(std::string_view name) const
 {
+    name = TTraits::resolveName(name);
    const auto & accessor = Traits::Accessor::instance();
    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
        return accessor.getValueString(*this, index);
@ -259,6 +263,7 @@ String BaseSettings<TTraits>::getString(std::string_view name) const
 template <typename TTraits>
 bool BaseSettings<TTraits>::tryGet(std::string_view name, Field & value) const
 {
+    name = TTraits::resolveName(name);
    const auto & accessor = Traits::Accessor::instance();
    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
    {
@ -276,6 +281,7 @@ bool BaseSettings<TTraits>::tryGet(std::string_view name, Field & value) const
 template <typename TTraits>
 bool BaseSettings<TTraits>::tryGetString(std::string_view name, String & value) const
 {
+    name = TTraits::resolveName(name);
    const auto & accessor = Traits::Accessor::instance();
    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
    {
@ -293,6 +299,7 @@ bool BaseSettings<TTraits>::tryGetString(std::string_view name, String & value)
 template <typename TTraits>
 bool BaseSettings<TTraits>::isChanged(std::string_view name) const
 {
+    name = TTraits::resolveName(name);
    const auto & accessor = Traits::Accessor::instance();
    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
        return accessor.isValueChanged(*this, index);
@ -345,6 +352,7 @@ void BaseSettings<TTraits>::resetToDefault()
 template <typename TTraits>
 void BaseSettings<TTraits>::resetToDefault(std::string_view name)
 {
+    name = TTraits::resolveName(name);
    const auto & accessor = Traits::Accessor::instance();
    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
        accessor.resetValueToDefault(*this, index);
@ -353,6 +361,7 @@ void BaseSettings<TTraits>::resetToDefault(std::string_view name)
 template <typename TTraits>
 bool BaseSettings<TTraits>::hasBuiltin(std::string_view name)
 {
+    name = TTraits::resolveName(name);
    const auto & accessor = Traits::Accessor::instance();
    return (accessor.find(name) != static_cast<size_t>(-1));
 }
@ -360,12 +369,14 @@ bool BaseSettings<TTraits>::hasBuiltin(std::string_view name)
 template <typename TTraits>
 bool BaseSettings<TTraits>::hasCustom(std::string_view name) const
 {
+    name = TTraits::resolveName(name);
    return tryGetCustomSetting(name);
 }

 template <typename TTraits>
 const char * BaseSettings<TTraits>::getTypeName(std::string_view name) const
 {
+    name = TTraits::resolveName(name);
    const auto & accessor = Traits::Accessor::instance();
    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
        return accessor.getTypeName(index);
@ -378,6 +389,7 @@ const char * BaseSettings<TTraits>::getTypeName(std::string_view name) const
 template <typename TTraits>
 const char * BaseSettings<TTraits>::getDescription(std::string_view name) const
 {
+    name = TTraits::resolveName(name);
    const auto & accessor = Traits::Accessor::instance();
    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
        return accessor.getDescription(index);
@ -390,18 +402,21 @@ const char * BaseSettings<TTraits>::getDescription(std::string_view name) const
 template <typename TTraits>
 void BaseSettings<TTraits>::checkCanSet(std::string_view name, const Field & value)
 {
+    name = TTraits::resolveName(name);
    castValueUtil(name, value);
 }

 template <typename TTraits>
 void BaseSettings<TTraits>::checkCanSetString(std::string_view name, const String & str)
 {
+    name = TTraits::resolveName(name);
    stringToValueUtil(name, str);
 }

 template <typename TTraits>
 Field BaseSettings<TTraits>::castValueUtil(std::string_view name, const Field & value)
 {
+    name = TTraits::resolveName(name);
    const auto & accessor = Traits::Accessor::instance();
    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
        return accessor.castValueUtil(index, value);
@ -414,6 +429,7 @@ Field BaseSettings<TTraits>::castValueUtil(std::string_view name, const Field &
 template <typename TTraits>
 String BaseSettings<TTraits>::valueToStringUtil(std::string_view name, const Field & value)
 {
+    name = TTraits::resolveName(name);
    const auto & accessor = Traits::Accessor::instance();
    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
        return accessor.valueToStringUtil(index, value);
@ -426,6 +442,7 @@ String BaseSettings<TTraits>::valueToStringUtil(std::string_view name, const Fie
 template <typename TTraits>
 Field BaseSettings<TTraits>::stringToValueUtil(std::string_view name, const String & str)
 {
+    name = TTraits::resolveName(name);
    try
    {
        const auto & accessor = Traits::Accessor::instance();
@ -539,30 +556,49 @@ String BaseSettings<TTraits>::toString() const
 template <typename TTraits>
 void BaseSettings<TTraits>::addProgramOptions(boost::program_options::options_description & options)
 {
+    const auto & settings_to_aliases = TTraits::settingsToAliases();
    for (const auto & field : all())
-        addProgramOption(options, field);
+    {
+        std::string_view name = field.getName();
+        addProgramOption(options, name, field);
+
+        if (auto it = settings_to_aliases.find(name); it != settings_to_aliases.end())
+        {
+            for (const auto alias : it->second)
+                addProgramOption(options, alias, field);
+        }
+    }
 }

 template <typename TTraits>
 void BaseSettings<TTraits>::addProgramOptionsAsMultitokens(boost::program_options::options_description & options)
 {
+    const auto & settings_to_aliases = TTraits::settingsToAliases();
    for (const auto & field : all())
-        addProgramOptionAsMultitoken(options, field);
+    {
+        std::string_view name = field.getName();
+        addProgramOptionAsMultitoken(options, name, field);
+
+        if (auto it = settings_to_aliases.find(name); it != settings_to_aliases.end())
+        {
+            for (const auto alias : it->second)
+                addProgramOptionAsMultitoken(options, alias, field);
+        }
+    }
 }

+
 template <typename TTraits>
-void BaseSettings<TTraits>::addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field)
+void BaseSettings<TTraits>::addProgramOption(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field)
 {
-    const std::string_view name = field.getName();
    auto on_program_option = boost::function1<void, const std::string &>([this, name](const std::string & value) { set(name, value); });
    options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
        name.data(), boost::program_options::value<std::string>()->composing()->notifier(on_program_option), field.getDescription())));
 }

 template <typename TTraits>
-void BaseSettings<TTraits>::addProgramOptionAsMultitoken(boost::program_options::options_description & options, const SettingFieldRef & field)
+void BaseSettings<TTraits>::addProgramOptionAsMultitoken(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field)
 {
-    const std::string_view name = field.getName();
    auto on_program_option = boost::function1<void, const Strings &>([this, name](const Strings & values) { set(name, values.back()); });
    options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
        name.data(), boost::program_options::value<Strings>()->multitoken()->composing()->notifier(on_program_option), field.getDescription())));
@ -825,6 +861,8 @@ bool BaseSettings<TTraits>::SettingFieldRef::isObsolete() const
    return accessor->isObsolete(index);
 }

+using AliasMap = std::unordered_map<std::string_view, std::string_view>;
+
 /// NOLINTNEXTLINE
 #define DECLARE_SETTINGS_TRAITS(SETTINGS_TRAITS_NAME, LIST_OF_SETTINGS_MACRO) \
    DECLARE_SETTINGS_TRAITS_COMMON(SETTINGS_TRAITS_NAME, LIST_OF_SETTINGS_MACRO, 0)
@ -839,7 +877,7 @@ bool BaseSettings<TTraits>::SettingFieldRef::isObsolete() const
    { \
        struct Data \
        { \
-            LIST_OF_SETTINGS_MACRO(DECLARE_SETTINGS_TRAITS_) \
+            LIST_OF_SETTINGS_MACRO(DECLARE_SETTINGS_TRAITS_, SKIP_ALIAS) \
        }; \
        \
        class Accessor \
@ -890,8 +928,60 @@ bool BaseSettings<TTraits>::SettingFieldRef::isObsolete() const
            std::unordered_map<std::string_view, size_t> name_to_index_map; \
        }; \
        static constexpr bool allow_custom_settings = ALLOW_CUSTOM_SETTINGS; \
+        \
+        static inline const AliasMap aliases_to_settings = \
+            DefineAliases() LIST_OF_SETTINGS_MACRO(ALIAS_TO, ALIAS_FROM); \
+        \
+        using SettingsToAliasesMap = std::unordered_map<std::string_view, std::vector<std::string_view>>; \
+        static inline const SettingsToAliasesMap & settingsToAliases() \
+        { \
+            static SettingsToAliasesMap setting_to_aliases_mapping = [] \
+            { \
+                std::unordered_map<std::string_view, std::vector<std::string_view>> map; \
+                for (const auto & [alias, destination] : aliases_to_settings) \
+                    map[destination].push_back(alias); \
+                return map; \
+            }(); \
+            return setting_to_aliases_mapping; \
+        } \
+        \
+        static std::string_view resolveName(std::string_view name) \
+        { \
+            if (auto it = aliases_to_settings.find(name); it != aliases_to_settings.end()) \
+                return it->second; \
+            return name; \
+        } \
    };

+
+/// NOLINTNEXTLINE
+#define SKIP_ALIAS(ALIAS_NAME)
+/// NOLINTNEXTLINE
+#define ALIAS_TO(TYPE, NAME, DEFAULT, DESCRIPTION, FLAGS) .setName(#NAME)
+/// NOLINTNEXTLINE
+#define ALIAS_FROM(ALIAS) .addAlias(#ALIAS)
+
+struct DefineAliases
+{
+    std::string_view name;
+    AliasMap map;
+
+    DefineAliases & setName(std::string_view value)
+    {
+        name = value;
+        return *this;
+    }
+
+    DefineAliases & addAlias(std::string_view value)
+    {
+        map.emplace(value, name);
+        return *this;
+    }
+
+    /// NOLINTNEXTLINE(google-explicit-constructor)
+    operator AliasMap() { return std::move(map); }
+};
+
 /// NOLINTNEXTLINE
 #define DECLARE_SETTINGS_TRAITS_(TYPE, NAME, DEFAULT, DESCRIPTION, FLAGS) \
    SettingField##TYPE NAME {DEFAULT};
@ -905,7 +995,7 @@ bool BaseSettings<TTraits>::SettingFieldRef::isObsolete() const
            Accessor res; \
            constexpr int IMPORTANT = 0x01; \
            UNUSED(IMPORTANT); \
-            LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_TRAITS_) \
+            LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_TRAITS_, SKIP_ALIAS) \
            for (size_t i : collections::range(res.field_infos.size())) \
            { \
                const auto & info = res.field_infos[i]; \
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@ -31,7 +31,7 @@ class IColumn;
  * to work correctly.
  */

-#define COMMON_SETTINGS(M) \
+#define COMMON_SETTINGS(M, ALIAS) \
    M(Dialect, dialect, Dialect::clickhouse, "Which dialect will be used to parse query", 0)\
    M(UInt64, min_compress_block_size, 65536, "The actual size of the block to compress, if the uncompressed data less than max_compress_block_size is no less than this value and no less than the volume of data for one mark.", 0) \
    M(UInt64, max_compress_block_size, 1048576, "The maximum size of blocks of uncompressed data before compressing for writing to a table.", 0) \
@ -115,7 +115,7 @@ class IColumn;
    M(Bool, optimize_move_to_prewhere, true, "Allows disabling WHERE to PREWHERE optimization in SELECT queries from MergeTree.", 0) \
    M(Bool, optimize_move_to_prewhere_if_final, false, "If query has `FINAL`, the optimization `move_to_prewhere` is not always correct and it is enabled only if both settings `optimize_move_to_prewhere` and `optimize_move_to_prewhere_if_final` are turned on", 0) \
    \
-    M(UInt64, replication_alter_partitions_sync, 1, "Wait for actions to manipulate the partitions. 0 - do not wait, 1 - wait for execution only of itself, 2 - wait for everyone.", 0) \
+    M(UInt64, alter_sync, 1, "Wait for actions to manipulate the partitions. 0 - do not wait, 1 - wait for execution only of itself, 2 - wait for everyone.", 0) ALIAS(replication_alter_partitions_sync) \
    M(Int64, replication_wait_for_inactive_replica_timeout, 120, "Wait for inactive replica to execute ALTER/OPTIMIZE. Time in seconds, 0 - do not wait, negative - wait for unlimited time.", 0) \
    \
    M(LoadBalancing, load_balancing, LoadBalancing::RANDOM, "Which replicas (among healthy replicas) to preferably send a query to (on the first attempt) for distributed processing.", 0) \
@ -551,6 +551,7 @@ class IColumn;
    \
    M(Bool, optimize_rewrite_sum_if_to_count_if, false, "Rewrite sumIf() and sum(if()) function countIf() function when logically equivalent", 0) \
    M(Bool, optimize_rewrite_aggregate_function_with_if, true, "Rewrite aggregate functions with if expression as argument when logically equivalent. For example, avg(if(cond, col, null)) can be rewritten to avgIf(cond, col)", 0) \
+    M(Bool, optimize_rewrite_array_exists_to_has, true, "Rewrite arrayExists() functions to has() when logically equivalent. For example, arrayExists(x -> x = 1, arr) can be rewritten to has(arr, 1)", 0) \
    M(UInt64, insert_shard_id, 0, "If non zero, when insert into a distributed table, the data will be inserted into the shard `insert_shard_id` synchronously. Possible values range from 1 to `shards_number` of corresponding distributed table", 0) \
    \
    M(Bool, collect_hash_table_stats_during_aggregation, true, "Enable collecting hash table statistics to optimize memory allocation", 0) \
@ -568,7 +569,7 @@ class IColumn;
    M(Bool, engine_file_allow_create_multiple_files, false, "Enables or disables creating a new file on each insert in file engine tables if format has suffix.", 0) \
    M(Bool, allow_experimental_database_replicated, false, "Allow to create databases with Replicated engine", 0) \
    M(UInt64, database_replicated_initial_query_timeout_sec, 300, "How long initial DDL query should wait for Replicated database to precess previous DDL queue entries", 0) \
-    M(Bool, database_replicated_enforce_synchronous_settings, false, "Enforces synchronous waiting for some queries (see also database_atomic_wait_for_drop_and_detach_synchronously, mutation_sync, replication_alter_partitions_sync). Not recommended to enable these settings.", 0) \
+    M(Bool, database_replicated_enforce_synchronous_settings, false, "Enforces synchronous waiting for some queries (see also database_atomic_wait_for_drop_and_detach_synchronously, mutation_sync, alter_sync). Not recommended to enable these settings.", 0) \
    M(UInt64, max_distributed_depth, 5, "Maximum distributed query depth", 0) \
    M(Bool, database_replicated_always_detach_permanently, false, "Execute DETACH TABLE as DETACH TABLE PERMANENTLY if database engine is Replicated", 0) \
    M(Bool, database_replicated_allow_only_replicated_engine, false, "Allow to create only Replicated tables in database with engine Replicated", 0) \
@ -708,7 +709,7 @@ class IColumn;
 #define MAKE_OBSOLETE(M, TYPE, NAME, DEFAULT) \
    M(TYPE, NAME, DEFAULT, "Obsolete setting, does nothing.", BaseSettingsHelpers::Flags::OBSOLETE)

-#define OBSOLETE_SETTINGS(M) \
+#define OBSOLETE_SETTINGS(M, ALIAS) \
    /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
    MAKE_OBSOLETE(M, UInt64, max_memory_usage_for_all_queries, 0) \
    MAKE_OBSOLETE(M, UInt64, multiple_joins_rewriter_version, 0) \
@ -746,7 +747,7 @@ class IColumn;
    /** The section above is for obsolete settings. Do not add anything there. */


-#define FORMAT_FACTORY_SETTINGS(M) \
+#define FORMAT_FACTORY_SETTINGS(M, ALIAS) \
    M(Char, format_csv_delimiter, ',', "The character to be considered as a delimiter in CSV data. If setting with a string, a string has to have a length of 1.", 0) \
    M(Bool, format_csv_allow_single_quotes, false, "If it is set to true, allow strings in single quotes.", 0) \
    M(Bool, format_csv_allow_double_quotes, true, "If it is set to true, allow strings in double quotes.", 0) \
@ -910,10 +911,10 @@ class IColumn;
 // End of FORMAT_FACTORY_SETTINGS
 // Please add settings non-related to formats into the COMMON_SETTINGS above.

-#define LIST_OF_SETTINGS(M)    \
-    COMMON_SETTINGS(M)         \
-    OBSOLETE_SETTINGS(M)       \
-    FORMAT_FACTORY_SETTINGS(M)
+#define LIST_OF_SETTINGS(M, ALIAS)    \
+    COMMON_SETTINGS(M, ALIAS)         \
+    OBSOLETE_SETTINGS(M, ALIAS)       \
+    FORMAT_FACTORY_SETTINGS(M, ALIAS)

 DECLARE_SETTINGS_TRAITS_ALLOW_CUSTOM_SETTINGS(SettingsTraits, LIST_OF_SETTINGS)

@ -923,8 +924,6 @@ DECLARE_SETTINGS_TRAITS_ALLOW_CUSTOM_SETTINGS(SettingsTraits, LIST_OF_SETTINGS)
  */
 struct Settings : public BaseSettings<SettingsTraits>, public IHints<2, Settings>
 {
-    /// For initialization from empty initializer-list to be "value initialization", not "aggregate initialization" in C++14.
-    /// http://en.cppreference.com/w/cpp/language/aggregate_initialization
    Settings() = default;

    /** Set multiple settings from "profile" (in server configuration file (users.xml), profiles contain groups of multiple settings).
--- a/src/Daemon/BaseDaemon.cpp
+++ b/src/Daemon/BaseDaemon.cpp
@ -994,6 +994,11 @@ void BaseDaemon::setupWatchdog()
    if (argv0)
        original_process_name = argv0;

+    bool restart = false;
+    const char * env_watchdog_restart = getenv("CLICKHOUSE_WATCHDOG_RESTART"); // NOLINT(concurrency-mt-unsafe)
+    if (env_watchdog_restart && 0 == strcmp(env_watchdog_restart, "1"))
+        restart = true;
+
    while (true)
    {
        /// This pipe is used to synchronize notifications to the service manager from the child process
@ -1141,14 +1146,14 @@ void BaseDaemon::setupWatchdog()
            logger().fatal("Child process was not exited normally by unknown reason.");
        }

-        /// Automatic restart is not enabled but you can play with it.
-#if 1
-        _exit(WEXITSTATUS(status));
-#else
-        logger().information("Will restart.");
-        if (argv0)
-            memcpy(argv0, original_process_name.c_str(), original_process_name.size());
-#endif
+        if (restart)
+        {
+            logger().information("Will restart.");
+            if (argv0)
+                memcpy(argv0, original_process_name.c_str(), original_process_name.size());
+        }
+        else
+            _exit(WEXITSTATUS(status));
    }
 }

--- a/src/Databases/DatabaseReplicatedSettings.h
+++ b/src/Databases/DatabaseReplicatedSettings.h
@ -7,7 +7,7 @@ namespace DB

 class ASTStorage;

-#define LIST_OF_DATABASE_REPLICATED_SETTINGS(M) \
+#define LIST_OF_DATABASE_REPLICATED_SETTINGS(M, ALIAS) \
    M(Float,  max_broken_tables_ratio, 0.5, "Do not recover replica automatically if the ratio of staled tables to all tables is greater", 0) \
    M(UInt64, max_replication_lag_to_enqueue, 10, "Replica will throw exception on attempt to execute query if its replication lag greater", 0) \
    M(UInt64, wait_entry_commited_timeout_sec, 3600, "Replicas will try to cancel query if timeout exceed, but initiator host has not executed it yet", 0) \
--- a/src/Databases/MySQL/ConnectionMySQLSettings.h
+++ b/src/Databases/MySQL/ConnectionMySQLSettings.h
@ -11,16 +11,16 @@ namespace DB

 class ASTStorage;

-#define LIST_OF_CONNECTION_MYSQL_SETTINGS(M) \
+#define LIST_OF_CONNECTION_MYSQL_SETTINGS(M, ALIAS) \
    M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precision are seen as String on ClickHouse's side.", 0) \

 /// Settings that should not change after the creation of a database.
 #define APPLY_FOR_IMMUTABLE_CONNECTION_MYSQL_SETTINGS(M) \
    M(mysql_datatypes_support_level)

-#define LIST_OF_MYSQL_DATABASE_SETTINGS(M) \
-    LIST_OF_CONNECTION_MYSQL_SETTINGS(M) \
-    LIST_OF_MYSQL_SETTINGS(M)
+#define LIST_OF_MYSQL_DATABASE_SETTINGS(M, ALIAS) \
+    LIST_OF_CONNECTION_MYSQL_SETTINGS(M, ALIAS) \
+    LIST_OF_MYSQL_SETTINGS(M, ALIAS)

 DECLARE_SETTINGS_TRAITS(ConnectionMySQLSettingsTraits, LIST_OF_MYSQL_DATABASE_SETTINGS)

--- a/src/Databases/MySQL/MaterializedMySQLSettings.h
+++ b/src/Databases/MySQL/MaterializedMySQLSettings.h
@ -8,7 +8,7 @@ namespace DB

 class ASTStorage;

-#define LIST_OF_MATERIALIZE_MODE_SETTINGS(M) \
+#define LIST_OF_MATERIALIZE_MODE_SETTINGS(M, ALIAS) \
    M(UInt64, max_rows_in_buffer, DEFAULT_BLOCK_SIZE, "Max rows that data is allowed to cache in memory(for single table and the cache data unable to query). when rows is exceeded, the data will be materialized", 0) \
    M(UInt64, max_bytes_in_buffer, DBMS_DEFAULT_BUFFER_SIZE, "Max bytes that data is allowed to cache in memory(for single table and the cache data unable to query). when rows is exceeded, the data will be materialized", 0) \
    M(UInt64, max_rows_in_buffers, DEFAULT_BLOCK_SIZE, "Max rows that data is allowed to cache in memory(for database and the cache data unable to query). when rows is exceeded, the data will be materialized", 0) \
--- a/src/Databases/SQLite/DatabaseSQLite.cpp
+++ b/src/Databases/SQLite/DatabaseSQLite.cpp
@ -58,7 +58,7 @@ DatabaseTablesIteratorPtr DatabaseSQLite::getTablesIterator(ContextPtr local_con
 }


-std::unordered_set<std::string> DatabaseSQLite::fetchTablesList() const
+NameSet DatabaseSQLite::fetchTablesList() const
 {
    if (!sqlite_db)
        sqlite_db = openSQLiteDB(database_path, getContext(), /* throw_on_error */true);
--- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
+++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
@ -17,6 +17,7 @@ namespace ErrorCodes
    extern const int FILE_DOESNT_EXIST;
    extern const int BAD_FILE_TYPE;
    extern const int FILE_ALREADY_EXISTS;
+    extern const int CANNOT_PARSE_INPUT_ASSERTION_FAILED;
 }

 DiskObjectStorageTransaction::DiskObjectStorageTransaction(
@ -275,8 +276,15 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp
                if (e.code() == ErrorCodes::UNKNOWN_FORMAT
                    || e.code() == ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF
                    || e.code() == ErrorCodes::CANNOT_READ_ALL_DATA
-                    || e.code() == ErrorCodes::CANNOT_OPEN_FILE)
+                    || e.code() == ErrorCodes::CANNOT_OPEN_FILE
+                    || e.code() == ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED)
                {
+                    LOG_DEBUG(
+                        &Poco::Logger::get("RemoveRecursiveObjectStorageOperation"),
+                        "Can't read metadata because of an exception. Just remove it from the filesystem. Path: {}, exception: {}",
+                        metadata_storage.getPath() + path_to_remove,
+                        e.message());
+
                    tx->unlinkFile(path_to_remove);
                }
                else
--- a/src/Functions/DateTimeTransforms.h
+++ b/src/Functions/DateTimeTransforms.h
@ -710,6 +710,56 @@ struct ToYearImpl
    using FactorTransform = ZeroTransform;
 };

+struct ToWeekYearImpl
+{
+    static constexpr auto name = "toWeekYear";
+
+    static constexpr Int8 week_mode = 3;
+
+    static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toYearWeek(t, week_mode).first;
+    }
+    static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toYearWeek(t, week_mode).first;
+    }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toYearWeek(ExtendedDayNum(d), week_mode).first;
+    }
+    static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toYearWeek(DayNum(d), week_mode).first;
+    }
+
+    using FactorTransform = ZeroTransform;
+};
+
+struct ToWeekOfWeekYearImpl
+{
+    static constexpr auto name = "toWeekOfWeekYear";
+
+    static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toISOWeek(t);
+    }
+    static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toISOWeek(t);
+    }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toISOWeek(ExtendedDayNum(d));
+    }
+    static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toISOWeek(DayNum(d));
+    }
+
+    using FactorTransform = ZeroTransform;
+};
+
 struct ToQuarterImpl
 {
    static constexpr auto name = "toQuarter";
--- a/src/Functions/formatDateTime.cpp
+++ b/src/Functions/formatDateTime.cpp
@ -527,6 +527,18 @@ private:
                return writeNumberWithPadding(dest, year, min_represent_digits);
        }

+        static size_t jodaWeekYear(size_t min_represent_digits, char * dest, Time source, UInt64, UInt32, const DateLUTImpl & timezone)
+        {
+            auto week_year = ToWeekYearImpl::execute(source, timezone);
+            return writeNumberWithPadding(dest, week_year, min_represent_digits);
+        }
+
+        static size_t jodaWeekOfWeekYear(size_t min_represent_digits, char * dest, Time source, UInt64, UInt32, const DateLUTImpl & timezone)
+        {
+            auto week_of_weekyear = ToWeekOfWeekYearImpl::execute(source, timezone);
+            return writeNumberWithPadding(dest, week_of_weekyear, min_represent_digits);
+        }
+
        static size_t jodaDayOfYear(size_t min_represent_digits, char * dest, Time source, UInt64, UInt32, const DateLUTImpl & timezone)
        {
            auto day_of_year = ToDayOfYearImpl::execute(source, timezone);
@ -597,6 +609,30 @@ private:
            return writeNumberWithPadding(dest, second_of_minute, min_represent_digits);
        }

+        static size_t jodaFractionOfSecond(size_t min_represent_digits, char * dest, Time /*source*/, UInt64 fractional_second, UInt32 scale, const DateLUTImpl & /*timezone*/)
+        {
+            if (min_represent_digits > 9)
+                min_represent_digits = 9;
+            if (fractional_second == 0)
+            {
+                for (UInt64 i = 0; i < min_represent_digits; ++i)
+                    dest[i] = '0';
+                return min_represent_digits;
+            }
+            auto str = toString(fractional_second);
+            if (min_represent_digits > scale)
+            {
+                for (UInt64 i = 0; i < min_represent_digits - scale; ++i)
+                    str += '0';
+            }
+            else if (min_represent_digits < scale)
+            {
+                str = str.substr(0, min_represent_digits);
+            }
+            memcpy(dest, str.data(), str.size());
+            return min_represent_digits;
+        }
+
        static size_t jodaTimezone(size_t min_represent_digits, char * dest, Time /*source*/, UInt64, UInt32, const DateLUTImpl & timezone)
        {
            if (min_represent_digits <= 3)
@ -1147,9 +1183,15 @@ public:
                        reserve_size += repetitions == 2 ? 2 : std::max(repetitions, 4);
                        break;
                    case 'x':
-                        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "format is not supported for WEEK_YEAR");
+                        instructions.emplace_back(std::bind_front(&Action<T>::jodaWeekYear, repetitions));
+                        /// weekyear range [1900, 2299]
+                        reserve_size += std::max(repetitions, 4);
+                        break;
                    case 'w':
-                        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "format is not supported for WEEK_OF_WEEK_YEAR");
+                        instructions.emplace_back(std::bind_front(&Action<T>::jodaWeekOfWeekYear, repetitions));
+                        /// Week of weekyear range [1, 52]
+                        reserve_size += std::max(repetitions, 2);
+                        break;
                    case 'e':
                        instructions.emplace_back(std::bind_front(&Action<T>::jodaDayOfWeek1Based, repetitions));
                        /// Day of week range [1, 7]
@ -1234,7 +1276,11 @@ public:
                        reserve_size += std::max(repetitions, 2);
                        break;
                    case 'S':
-                        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "format is not supported for FRACTION_OF_SECOND");
+                        /// Default fraction of second is 0
+                        instructions.emplace_back(std::bind_front(&Action<T>::jodaFractionOfSecond, repetitions));
+                        /// 'S' repetitions range [0, 9]
+                        reserve_size += repetitions <= 9 ? repetitions : 9;
+                        break;
                    case 'z':
                        if (repetitions <= 3)
                            throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Short name time zone is not yet supported");
--- a/src/IO/S3/AWSLogger.cpp
+++ b/src/IO/S3/AWSLogger.cpp
@ -0,0 +1,75 @@
+#include <IO/S3/AWSLogger.h>
+
+#if USE_AWS_S3
+
+#include <aws/core/utils/logging/LogLevel.h>
+
+namespace
+{
+
+const char * S3_LOGGER_TAG_NAMES[][2] = {
+    {"AWSClient", "AWSClient"},
+    {"AWSAuthV4Signer", "AWSClient (AWSAuthV4Signer)"},
+};
+
+const std::pair<DB::LogsLevel, Poco::Message::Priority> & convertLogLevel(Aws::Utils::Logging::LogLevel log_level)
+{
+    /// We map levels to our own logger 1 to 1 except WARN+ levels. In most cases we failover such errors with retries
+    /// and don't want to see them as Errors in our logs.
+    static const std::unordered_map<Aws::Utils::Logging::LogLevel, std::pair<DB::LogsLevel, Poco::Message::Priority>> mapping =
+    {
+        {Aws::Utils::Logging::LogLevel::Off, {DB::LogsLevel::none, Poco::Message::PRIO_INFORMATION}},
+        {Aws::Utils::Logging::LogLevel::Fatal, {DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION}},
+        {Aws::Utils::Logging::LogLevel::Error, {DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION}},
+        {Aws::Utils::Logging::LogLevel::Warn, {DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION}},
+        {Aws::Utils::Logging::LogLevel::Info, {DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION}},
+        {Aws::Utils::Logging::LogLevel::Debug, {DB::LogsLevel::debug, Poco::Message::PRIO_TEST}},
+        {Aws::Utils::Logging::LogLevel::Trace, {DB::LogsLevel::trace, Poco::Message::PRIO_TEST}},
+    };
+    return mapping.at(log_level);
+}
+
+}
+
+namespace DB::S3
+{
+
+AWSLogger::AWSLogger(bool enable_s3_requests_logging_)
+    : enable_s3_requests_logging(enable_s3_requests_logging_)
+{
+    for (auto [tag, name] : S3_LOGGER_TAG_NAMES)
+        tag_loggers[tag] = &Poco::Logger::get(name);
+
+    default_logger = tag_loggers[S3_LOGGER_TAG_NAMES[0][0]];
+}
+
+Aws::Utils::Logging::LogLevel AWSLogger::GetLogLevel() const
+{
+    if (enable_s3_requests_logging)
+        return Aws::Utils::Logging::LogLevel::Trace;
+    else
+        return Aws::Utils::Logging::LogLevel::Info;
+}
+
+void AWSLogger::Log(Aws::Utils::Logging::LogLevel log_level, const char * tag, const char * format_str, ...) // NOLINT
+{
+    callLogImpl(log_level, tag, format_str); /// FIXME. Variadic arguments?
+}
+
+void AWSLogger::LogStream(Aws::Utils::Logging::LogLevel log_level, const char * tag, const Aws::OStringStream & message_stream)
+{
+    callLogImpl(log_level, tag, message_stream.str().c_str());
+}
+
+void AWSLogger::callLogImpl(Aws::Utils::Logging::LogLevel log_level, const char * tag, const char * message)
+{
+    const auto & [level, prio] = convertLogLevel(log_level);
+    if (tag_loggers.contains(tag))
+        LOG_IMPL(tag_loggers[tag], level, prio, fmt::runtime(message));
+    else
+        LOG_IMPL(default_logger, level, prio, "{}: {}", tag, message);
+}
+
+}
+
+#endif
--- a/src/IO/S3/AWSLogger.h
+++ b/src/IO/S3/AWSLogger.h
@ -0,0 +1,37 @@
+#pragma once
+
+#include "config.h"
+
+#if USE_AWS_S3
+#include <aws/core/utils/logging/LogSystemInterface.h>
+
+#include <Common/logger_useful.h>
+
+namespace DB::S3
+{
+class AWSLogger final : public Aws::Utils::Logging::LogSystemInterface
+{
+public:
+    explicit AWSLogger(bool enable_s3_requests_logging_);
+
+    ~AWSLogger() final = default;
+
+    Aws::Utils::Logging::LogLevel GetLogLevel() const final;
+
+    void Log(Aws::Utils::Logging::LogLevel log_level, const char * tag, const char * format_str, ...) final; // NOLINT
+
+    void LogStream(Aws::Utils::Logging::LogLevel log_level, const char * tag, const Aws::OStringStream & message_stream) final;
+
+    void callLogImpl(Aws::Utils::Logging::LogLevel log_level, const char * tag, const char * message);
+
+    void Flush() final {}
+
+private:
+    Poco::Logger * default_logger;
+    bool enable_s3_requests_logging;
+    std::unordered_map<String, Poco::Logger *> tag_loggers;
+};
+
+}
+
+#endif
--- a/src/IO/S3/Client.cpp
+++ b/src/IO/S3/Client.cpp
@ -9,9 +9,13 @@
 #include <aws/s3/model/ListObjectsV2Request.h>
 #include <aws/core/client/AWSErrorMarshaller.h>
 #include <aws/core/endpoint/EndpointParameter.h>
+#include <aws/core/utils/HashingUtils.h>

 #include <IO/S3Common.h>
 #include <IO/S3/Requests.h>
+#include <IO/S3/PocoHTTPClientFactory.h>
+#include <IO/S3/AWSLogger.h>
+#include <IO/S3/Credentials.h>

 #include <Common/assert_cast.h>

@ -393,6 +397,93 @@ void ClientCacheRegistry::clearCacheForAll()

 }

+ClientFactory::ClientFactory()
+{
+    aws_options = Aws::SDKOptions{};
+    Aws::InitAPI(aws_options);
+    Aws::Utils::Logging::InitializeAWSLogging(std::make_shared<AWSLogger>(false));
+    Aws::Http::SetHttpClientFactory(std::make_shared<PocoHTTPClientFactory>());
+}
+
+ClientFactory::~ClientFactory()
+{
+    Aws::Utils::Logging::ShutdownAWSLogging();
+    Aws::ShutdownAPI(aws_options);
+}
+
+ClientFactory & ClientFactory::instance()
+{
+    static ClientFactory ret;
+    return ret;
+}
+
+std::unique_ptr<S3::Client> ClientFactory::create( // NOLINT
+    const PocoHTTPClientConfiguration & cfg_,
+    bool is_virtual_hosted_style,
+    const String & access_key_id,
+    const String & secret_access_key,
+    const String & server_side_encryption_customer_key_base64,
+    HTTPHeaderEntries headers,
+    bool use_environment_credentials,
+    bool use_insecure_imds_request)
+{
+    PocoHTTPClientConfiguration client_configuration = cfg_;
+    client_configuration.updateSchemeAndRegion();
+
+    if (!server_side_encryption_customer_key_base64.empty())
+    {
+        /// See Client::GeneratePresignedUrlWithSSEC().
+
+        headers.push_back({Aws::S3::SSEHeaders::SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM,
+            Aws::S3::Model::ServerSideEncryptionMapper::GetNameForServerSideEncryption(Aws::S3::Model::ServerSideEncryption::AES256)});
+
+        headers.push_back({Aws::S3::SSEHeaders::SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
+            server_side_encryption_customer_key_base64});
+
+        Aws::Utils::ByteBuffer buffer = Aws::Utils::HashingUtils::Base64Decode(server_side_encryption_customer_key_base64);
+        String str_buffer(reinterpret_cast<char *>(buffer.GetUnderlyingData()), buffer.GetLength());
+        headers.push_back({Aws::S3::SSEHeaders::SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
+            Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateMD5(str_buffer))});
+    }
+
+    client_configuration.extra_headers = std::move(headers);
+
+    Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key);
+    auto credentials_provider = std::make_shared<S3CredentialsProviderChain>(
+            client_configuration,
+            std::move(credentials),
+            use_environment_credentials,
+            use_insecure_imds_request);
+
+    client_configuration.retryStrategy = std::make_shared<Client::RetryStrategy>(std::move(client_configuration.retryStrategy));
+    return Client::create(
+        client_configuration.s3_max_redirects,
+        std::move(credentials_provider),
+        std::move(client_configuration), // Client configuration.
+        Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
+        is_virtual_hosted_style || client_configuration.endpointOverride.empty() /// Use virtual addressing if endpoint is not specified.
+    );
+}
+
+PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT
+    const String & force_region,
+    const RemoteHostFilter & remote_host_filter,
+    unsigned int s3_max_redirects,
+    bool enable_s3_requests_logging,
+    bool for_disk_s3,
+    const ThrottlerPtr & get_request_throttler,
+    const ThrottlerPtr & put_request_throttler)
+{
+    return PocoHTTPClientConfiguration(
+        force_region,
+        remote_host_filter,
+        s3_max_redirects,
+        enable_s3_requests_logging,
+        for_disk_s3,
+        get_request_throttler,
+        put_request_throttler);
+}
+
 }

 }
--- a/src/IO/S3/Client.h
+++ b/src/IO/S3/Client.h
@ -10,7 +10,9 @@

 #include <IO/S3/URI.h>
 #include <IO/S3/Requests.h>
+#include <IO/S3/PocoHTTPClient.h>

+#include <aws/core/Aws.h>
 #include <aws/core/client/DefaultRetryStrategy.h>
 #include <aws/s3/S3Client.h>
 #include <aws/s3/S3ServiceClientModel.h>
@ -302,6 +304,39 @@ private:
    Poco::Logger * log;
 };

+class ClientFactory
+{
+public:
+    ~ClientFactory();
+
+    static ClientFactory & instance();
+
+    std::unique_ptr<S3::Client> create(
+        const PocoHTTPClientConfiguration & cfg,
+        bool is_virtual_hosted_style,
+        const String & access_key_id,
+        const String & secret_access_key,
+        const String & server_side_encryption_customer_key_base64,
+        HTTPHeaderEntries headers,
+        bool use_environment_credentials,
+        bool use_insecure_imds_request);
+
+    PocoHTTPClientConfiguration createClientConfiguration(
+        const String & force_region,
+        const RemoteHostFilter & remote_host_filter,
+        unsigned int s3_max_redirects,
+        bool enable_s3_requests_logging,
+        bool for_disk_s3,
+        const ThrottlerPtr & get_request_throttler,
+        const ThrottlerPtr & put_request_throttler);
+
+private:
+    ClientFactory();
+
+    Aws::SDKOptions aws_options;
+    std::atomic<bool> s3_requests_logging_enabled;
+};
+
 }

 }
--- a/src/IO/S3/Credentials.cpp
+++ b/src/IO/S3/Credentials.cpp
@ -0,0 +1,522 @@
+#include <IO/S3/Credentials.h>
+
+#if USE_AWS_S3
+#    include <aws/core/Version.h>
+#    include <aws/core/platform/OSVersionInfo.h>
+#    include <aws/core/auth/STSCredentialsProvider.h>
+#    include <aws/core/platform/Environment.h>
+#    include <aws/core/client/SpecifiedRetryableErrorsRetryStrategy.h>
+#    include <aws/core/utils/json/JsonSerializer.h>
+#    include <aws/core/utils/UUID.h>
+#    include <aws/core/http/HttpClientFactory.h>
+
+#    include <Common/logger_useful.h>
+
+#    include <IO/S3/PocoHTTPClient.h>
+#    include <IO/S3/PocoHTTPClientFactory.h>
+#    include <IO/S3/Client.h>
+
+#    include <fstream>
+
+namespace DB::S3
+{
+
+AWSEC2MetadataClient::AWSEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration, const char * endpoint_)
+    : Aws::Internal::AWSHttpResourceClient(client_configuration)
+    , endpoint(endpoint_)
+    , logger(&Poco::Logger::get("AWSEC2InstanceProfileConfigLoader"))
+{
+}
+
+Aws::String AWSEC2MetadataClient::GetResource(const char * resource_path) const
+{
+    return GetResource(endpoint.c_str(), resource_path, nullptr/*authToken*/);
+}
+
+Aws::String AWSEC2MetadataClient::getDefaultCredentials() const
+{
+    String credentials_string;
+    {
+        std::lock_guard locker(token_mutex);
+
+        LOG_TRACE(logger, "Getting default credentials for ec2 instance from {}", endpoint);
+        auto result = GetResourceWithAWSWebServiceResult(endpoint.c_str(), EC2_SECURITY_CREDENTIALS_RESOURCE, nullptr);
+        credentials_string = result.GetPayload();
+        if (result.GetResponseCode() == Aws::Http::HttpResponseCode::UNAUTHORIZED)
+        {
+            return {};
+        }
+    }
+
+    String trimmed_credentials_string = Aws::Utils::StringUtils::Trim(credentials_string.c_str());
+    if (trimmed_credentials_string.empty())
+        return {};
+
+    std::vector<String> security_credentials = Aws::Utils::StringUtils::Split(trimmed_credentials_string, '\n');
+
+    LOG_DEBUG(logger, "Calling EC2MetadataService resource, {} returned credential string {}.",
+            EC2_SECURITY_CREDENTIALS_RESOURCE, trimmed_credentials_string);
+
+    if (security_credentials.empty())
+    {
+        LOG_WARNING(logger, "Initial call to EC2MetadataService to get credentials failed.");
+        return {};
+    }
+
+    Aws::StringStream ss;
+    ss << EC2_SECURITY_CREDENTIALS_RESOURCE << "/" << security_credentials[0];
+    LOG_DEBUG(logger, "Calling EC2MetadataService resource {}.", ss.str());
+    return GetResource(ss.str().c_str());
+}
+
+Aws::String AWSEC2MetadataClient::awsComputeUserAgentString()
+{
+    Aws::StringStream ss;
+    ss << "aws-sdk-cpp/" << Aws::Version::GetVersionString() << " " << Aws::OSVersionInfo::ComputeOSVersionString()
+            << " " << Aws::Version::GetCompilerVersionString();
+    return ss.str();
+}
+
+Aws::String AWSEC2MetadataClient::getDefaultCredentialsSecurely() const
+{
+    String user_agent_string = awsComputeUserAgentString();
+    String new_token;
+
+    {
+        std::lock_guard locker(token_mutex);
+
+        Aws::StringStream ss;
+        ss << endpoint << EC2_IMDS_TOKEN_RESOURCE;
+        std::shared_ptr<Aws::Http::HttpRequest> token_request(Aws::Http::CreateHttpRequest(ss.str(), Aws::Http::HttpMethod::HTTP_PUT,
+                                                                    Aws::Utils::Stream::DefaultResponseStreamFactoryMethod));
+        token_request->SetHeaderValue(EC2_IMDS_TOKEN_TTL_HEADER, EC2_IMDS_TOKEN_TTL_DEFAULT_VALUE);
+        token_request->SetUserAgent(user_agent_string);
+        LOG_TRACE(logger, "Calling EC2MetadataService to get token.");
+        auto result = GetResourceWithAWSWebServiceResult(token_request);
+        const String & token_string = result.GetPayload();
+        new_token = Aws::Utils::StringUtils::Trim(token_string.c_str());
+
+        if (result.GetResponseCode() == Aws::Http::HttpResponseCode::BAD_REQUEST)
+        {
+            return {};
+        }
+        else if (result.GetResponseCode() != Aws::Http::HttpResponseCode::OK || new_token.empty())
+        {
+            LOG_TRACE(logger, "Calling EC2MetadataService to get token failed, falling back to less secure way.");
+            return getDefaultCredentials();
+        }
+        token = new_token;
+    }
+
+    String url = endpoint + EC2_SECURITY_CREDENTIALS_RESOURCE;
+    std::shared_ptr<Aws::Http::HttpRequest> profile_request(Aws::Http::CreateHttpRequest(url,
+            Aws::Http::HttpMethod::HTTP_GET,
+            Aws::Utils::Stream::DefaultResponseStreamFactoryMethod));
+    profile_request->SetHeaderValue(EC2_IMDS_TOKEN_HEADER, new_token);
+    profile_request->SetUserAgent(user_agent_string);
+    String profile_string = GetResourceWithAWSWebServiceResult(profile_request).GetPayload();
+
+    String trimmed_profile_string = Aws::Utils::StringUtils::Trim(profile_string.c_str());
+    std::vector<String> security_credentials = Aws::Utils::StringUtils::Split(trimmed_profile_string, '\n');
+
+    LOG_DEBUG(logger, "Calling EC2MetadataService resource, {} with token returned profile string {}.",
+            EC2_SECURITY_CREDENTIALS_RESOURCE, trimmed_profile_string);
+
+    if (security_credentials.empty())
+    {
+        LOG_WARNING(logger, "Calling EC2Metadataservice to get profiles failed.");
+        return {};
+    }
+
+    Aws::StringStream ss;
+    ss << endpoint << EC2_SECURITY_CREDENTIALS_RESOURCE << "/" << security_credentials[0];
+    std::shared_ptr<Aws::Http::HttpRequest> credentials_request(Aws::Http::CreateHttpRequest(ss.str(),
+            Aws::Http::HttpMethod::HTTP_GET,
+            Aws::Utils::Stream::DefaultResponseStreamFactoryMethod));
+    credentials_request->SetHeaderValue(EC2_IMDS_TOKEN_HEADER, new_token);
+    credentials_request->SetUserAgent(user_agent_string);
+    LOG_DEBUG(logger, "Calling EC2MetadataService resource {} with token.", ss.str());
+    return GetResourceWithAWSWebServiceResult(credentials_request).GetPayload();
+}
+
+Aws::String AWSEC2MetadataClient::getCurrentRegion() const
+{
+    return Aws::Region::AWS_GLOBAL;
+}
+
+std::shared_ptr<AWSEC2MetadataClient> InitEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration)
+{
+    Aws::String ec2_metadata_service_endpoint = Aws::Environment::GetEnv("AWS_EC2_METADATA_SERVICE_ENDPOINT");
+    auto * logger = &Poco::Logger::get("AWSEC2InstanceProfileConfigLoader");
+    if (ec2_metadata_service_endpoint.empty())
+    {
+        Aws::String ec2_metadata_service_endpoint_mode = Aws::Environment::GetEnv("AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE");
+        if (ec2_metadata_service_endpoint_mode.length() == 0)
+        {
+            ec2_metadata_service_endpoint = "http://169.254.169.254"; //default to IPv4 default endpoint
+        }
+        else
+        {
+            if (ec2_metadata_service_endpoint_mode.length() == 4)
+            {
+                if (Aws::Utils::StringUtils::CaselessCompare(ec2_metadata_service_endpoint_mode.c_str(), "ipv4"))
+                {
+                    ec2_metadata_service_endpoint = "http://169.254.169.254"; //default to IPv4 default endpoint
+                }
+                else if (Aws::Utils::StringUtils::CaselessCompare(ec2_metadata_service_endpoint_mode.c_str(), "ipv6"))
+                {
+                    ec2_metadata_service_endpoint = "http://[fd00:ec2::254]";
+                }
+                else
+                {
+                    LOG_ERROR(logger, "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE can only be set to ipv4 or ipv6, received: {}", ec2_metadata_service_endpoint_mode);
+                }
+            }
+            else
+            {
+                LOG_ERROR(logger, "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE can only be set to ipv4 or ipv6, received: {}", ec2_metadata_service_endpoint_mode);
+            }
+        }
+    }
+    LOG_INFO(logger, "Using IMDS endpoint: {}", ec2_metadata_service_endpoint);
+    return std::make_shared<AWSEC2MetadataClient>(client_configuration, ec2_metadata_service_endpoint.c_str());
+}
+
+AWSEC2InstanceProfileConfigLoader::AWSEC2InstanceProfileConfigLoader(const std::shared_ptr<AWSEC2MetadataClient> & client_, bool use_secure_pull_)
+    : client(client_)
+    , use_secure_pull(use_secure_pull_)
+    , logger(&Poco::Logger::get("AWSEC2InstanceProfileConfigLoader"))
+{
+}
+
+bool AWSEC2InstanceProfileConfigLoader::LoadInternal()
+{
+    auto credentials_str = use_secure_pull ? client->getDefaultCredentialsSecurely() : client->getDefaultCredentials();
+
+    /// See EC2InstanceProfileConfigLoader.
+    if (credentials_str.empty())
+        return false;
+
+    Aws::Utils::Json::JsonValue credentials_doc(credentials_str);
+    if (!credentials_doc.WasParseSuccessful())
+    {
+        LOG_ERROR(logger, "Failed to parse output from EC2MetadataService.");
+        return false;
+    }
+    String access_key, secret_key, token;
+
+    auto credentials_view = credentials_doc.View();
+    access_key = credentials_view.GetString("AccessKeyId");
+    LOG_TRACE(logger, "Successfully pulled credentials from EC2MetadataService with access key.");
+
+    secret_key = credentials_view.GetString("SecretAccessKey");
+    token = credentials_view.GetString("Token");
+
+    auto region = client->getCurrentRegion();
+
+    Aws::Config::Profile profile;
+    profile.SetCredentials(Aws::Auth::AWSCredentials(access_key, secret_key, token));
+    profile.SetRegion(region);
+    profile.SetName(Aws::Config::INSTANCE_PROFILE_KEY);
+
+    m_profiles[Aws::Config::INSTANCE_PROFILE_KEY] = profile;
+
+    return true;
+}
+
+AWSInstanceProfileCredentialsProvider::AWSInstanceProfileCredentialsProvider(const std::shared_ptr<AWSEC2InstanceProfileConfigLoader> & config_loader)
+    : ec2_metadata_config_loader(config_loader)
+    , load_frequency_ms(Aws::Auth::REFRESH_THRESHOLD)
+    , logger(&Poco::Logger::get("AWSInstanceProfileCredentialsProvider"))
+{
+    LOG_INFO(logger, "Creating Instance with injected EC2MetadataClient and refresh rate.");
+}
+
+Aws::Auth::AWSCredentials AWSInstanceProfileCredentialsProvider::GetAWSCredentials()
+{
+    refreshIfExpired();
+    Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock);
+    auto profile_it = ec2_metadata_config_loader->GetProfiles().find(Aws::Config::INSTANCE_PROFILE_KEY);
+
+    if (profile_it != ec2_metadata_config_loader->GetProfiles().end())
+    {
+        return profile_it->second.GetCredentials();
+    }
+
+    return Aws::Auth::AWSCredentials();
+}
+
+void AWSInstanceProfileCredentialsProvider::Reload()
+{
+    LOG_INFO(logger, "Credentials have expired attempting to repull from EC2 Metadata Service.");
+    ec2_metadata_config_loader->Load();
+    AWSCredentialsProvider::Reload();
+}
+
+void AWSInstanceProfileCredentialsProvider::refreshIfExpired()
+{
+    LOG_DEBUG(logger, "Checking if latest credential pull has expired.");
+    Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock);
+    if (!IsTimeToRefresh(load_frequency_ms))
+    {
+        return;
+    }
+
+    guard.UpgradeToWriterLock();
+    if (!IsTimeToRefresh(load_frequency_ms)) // double-checked lock to avoid refreshing twice
+    {
+        return;
+    }
+    Reload();
+}
+
+AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider::AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider(DB::S3::PocoHTTPClientConfiguration & aws_client_configuration)
+    : logger(&Poco::Logger::get("AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider"))
+{
+    // check environment variables
+    String tmp_region = Aws::Environment::GetEnv("AWS_DEFAULT_REGION");
+    role_arn = Aws::Environment::GetEnv("AWS_ROLE_ARN");
+    token_file = Aws::Environment::GetEnv("AWS_WEB_IDENTITY_TOKEN_FILE");
+    session_name = Aws::Environment::GetEnv("AWS_ROLE_SESSION_NAME");
+
+    // check profile_config if either m_roleArn or m_tokenFile is not loaded from environment variable
+    // region source is not enforced, but we need it to construct sts endpoint, if we can't find from environment, we should check if it's set in config file.
+    if (role_arn.empty() || token_file.empty() || tmp_region.empty())
+    {
+        auto profile = Aws::Config::GetCachedConfigProfile(Aws::Auth::GetConfigProfileName());
+        if (tmp_region.empty())
+        {
+            tmp_region = profile.GetRegion();
+        }
+        // If either of these two were not found from environment, use whatever found for all three in config file
+        if (role_arn.empty() || token_file.empty())
+        {
+            role_arn = profile.GetRoleArn();
+            token_file = profile.GetValue("web_identity_token_file");
+            session_name = profile.GetValue("role_session_name");
+        }
+    }
+
+    if (token_file.empty())
+    {
+        LOG_WARNING(logger, "Token file must be specified to use STS AssumeRole web identity creds provider.");
+        return; // No need to do further constructing
+    }
+    else
+    {
+        LOG_DEBUG(logger, "Resolved token_file from profile_config or environment variable to be {}", token_file);
+    }
+
+    if (role_arn.empty())
+    {
+        LOG_WARNING(logger, "RoleArn must be specified to use STS AssumeRole web identity creds provider.");
+        return; // No need to do further constructing
+    }
+    else
+    {
+        LOG_DEBUG(logger, "Resolved role_arn from profile_config or environment variable to be {}", role_arn);
+    }
+
+    if (tmp_region.empty())
+    {
+        tmp_region = Aws::Region::US_EAST_1;
+    }
+    else
+    {
+        LOG_DEBUG(logger, "Resolved region from profile_config or environment variable to be {}", tmp_region);
+    }
+
+    if (session_name.empty())
+    {
+        session_name = Aws::Utils::UUID::RandomUUID();
+    }
+    else
+    {
+        LOG_DEBUG(logger, "Resolved session_name from profile_config or environment variable to be {}", session_name);
+    }
+
+    aws_client_configuration.scheme = Aws::Http::Scheme::HTTPS;
+    aws_client_configuration.region = tmp_region;
+
+    std::vector<String> retryable_errors;
+    retryable_errors.push_back("IDPCommunicationError");
+    retryable_errors.push_back("InvalidIdentityToken");
+
+    aws_client_configuration.retryStrategy = std::make_shared<Aws::Client::SpecifiedRetryableErrorsRetryStrategy>(
+        retryable_errors, /* maxRetries = */3);
+
+    client = std::make_unique<Aws::Internal::STSCredentialsClient>(aws_client_configuration);
+    initialized = true;
+    LOG_INFO(logger, "Creating STS AssumeRole with web identity creds provider.");
+}
+
+Aws::Auth::AWSCredentials AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider::GetAWSCredentials()
+{
+    // A valid client means required information like role arn and token file were constructed correctly.
+    // We can use this provider to load creds, otherwise, we can just return empty creds.
+    if (!initialized)
+    {
+        return Aws::Auth::AWSCredentials();
+    }
+    refreshIfExpired();
+    Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock);
+    return credentials;
+}
+
+void AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider::Reload()
+{
+    LOG_INFO(logger, "Credentials have expired, attempting to renew from STS.");
+
+    std::ifstream token_stream(token_file.data());
+    if (token_stream)
+    {
+        String token_string((std::istreambuf_iterator<char>(token_stream)), std::istreambuf_iterator<char>());
+        token = token_string;
+    }
+    else
+    {
+        LOG_INFO(logger, "Can't open token file: {}", token_file);
+        return;
+    }
+    Aws::Internal::STSCredentialsClient::STSAssumeRoleWithWebIdentityRequest request{session_name, role_arn, token};
+
+    auto result = client->GetAssumeRoleWithWebIdentityCredentials(request);
+    LOG_TRACE(logger, "Successfully retrieved credentials.");
+    credentials = result.creds;
+}
+
+void AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider::refreshIfExpired()
+{
+    Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock);
+    if (!credentials.IsExpiredOrEmpty())
+    {
+        return;
+    }
+
+    guard.UpgradeToWriterLock();
+    if (!credentials.IsExpiredOrEmpty()) // double-checked lock to avoid refreshing twice
+    {
+        return;
+    }
+
+    Reload();
+}
+
+S3CredentialsProviderChain::S3CredentialsProviderChain(
+        const DB::S3::PocoHTTPClientConfiguration & configuration,
+        const Aws::Auth::AWSCredentials & credentials,
+        bool use_environment_credentials,
+        bool use_insecure_imds_request)
+{
+    auto * logger = &Poco::Logger::get("S3CredentialsProviderChain");
+
+    /// add explicit credentials to the front of the chain
+    /// because it's manually defined by the user
+    if (!credentials.IsEmpty())
+    {
+        AddProvider(std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>(credentials));
+        return;
+    }
+
+    if (use_environment_credentials)
+    {
+        static const char AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI[] = "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI";
+        static const char AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI[] = "AWS_CONTAINER_CREDENTIALS_FULL_URI";
+        static const char AWS_ECS_CONTAINER_AUTHORIZATION_TOKEN[] = "AWS_CONTAINER_AUTHORIZATION_TOKEN";
+        static const char AWS_EC2_METADATA_DISABLED[] = "AWS_EC2_METADATA_DISABLED";
+
+        /// The only difference from DefaultAWSCredentialsProviderChain::DefaultAWSCredentialsProviderChain()
+        /// is that this chain uses custom ClientConfiguration. Also we removed process provider because it's useless in our case.
+        ///
+        /// AWS API tries credentials providers one by one. Some of providers (like ProfileConfigFileAWSCredentialsProvider) can be
+        /// quite verbose even if nobody configured them. So we use our provider first and only after it use default providers.
+        {
+            DB::S3::PocoHTTPClientConfiguration aws_client_configuration = DB::S3::ClientFactory::instance().createClientConfiguration(
+                configuration.region,
+                configuration.remote_host_filter,
+                configuration.s3_max_redirects,
+                configuration.enable_s3_requests_logging,
+                configuration.for_disk_s3,
+                configuration.get_request_throttler,
+                configuration.put_request_throttler);
+            AddProvider(std::make_shared<AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider>(aws_client_configuration));
+        }
+
+        AddProvider(std::make_shared<Aws::Auth::EnvironmentAWSCredentialsProvider>());
+
+
+        /// ECS TaskRole Credentials only available when ENVIRONMENT VARIABLE is set.
+        const auto relative_uri = Aws::Environment::GetEnv(AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI);
+        LOG_DEBUG(logger, "The environment variable value {} is {}", AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI,
+                relative_uri);
+
+        const auto absolute_uri = Aws::Environment::GetEnv(AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI);
+        LOG_DEBUG(logger, "The environment variable value {} is {}", AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI,
+                absolute_uri);
+
+        const auto ec2_metadata_disabled = Aws::Environment::GetEnv(AWS_EC2_METADATA_DISABLED);
+        LOG_DEBUG(logger, "The environment variable value {} is {}", AWS_EC2_METADATA_DISABLED,
+                ec2_metadata_disabled);
+
+        if (!relative_uri.empty())
+        {
+            AddProvider(std::make_shared<Aws::Auth::TaskRoleCredentialsProvider>(relative_uri.c_str()));
+            LOG_INFO(logger, "Added ECS metadata service credentials provider with relative path: [{}] to the provider chain.",
+                    relative_uri);
+        }
+        else if (!absolute_uri.empty())
+        {
+            const auto token = Aws::Environment::GetEnv(AWS_ECS_CONTAINER_AUTHORIZATION_TOKEN);
+            AddProvider(std::make_shared<Aws::Auth::TaskRoleCredentialsProvider>(absolute_uri.c_str(), token.c_str()));
+
+            /// DO NOT log the value of the authorization token for security purposes.
+            LOG_INFO(logger, "Added ECS credentials provider with URI: [{}] to the provider chain with a{} authorization token.",
+                    absolute_uri, token.empty() ? "n empty" : " non-empty");
+        }
+        else if (Aws::Utils::StringUtils::ToLower(ec2_metadata_disabled.c_str()) != "true")
+        {
+            DB::S3::PocoHTTPClientConfiguration aws_client_configuration = DB::S3::ClientFactory::instance().createClientConfiguration(
+                configuration.region,
+                configuration.remote_host_filter,
+                configuration.s3_max_redirects,
+                configuration.enable_s3_requests_logging,
+                configuration.for_disk_s3,
+                configuration.get_request_throttler,
+                configuration.put_request_throttler);
+
+            /// See MakeDefaultHttpResourceClientConfiguration().
+            /// This is part of EC2 metadata client, but unfortunately it can't be accessed from outside
+            /// of contrib/aws/aws-cpp-sdk-core/source/internal/AWSHttpResourceClient.cpp
+            aws_client_configuration.maxConnections = 2;
+            aws_client_configuration.scheme = Aws::Http::Scheme::HTTP;
+
+            /// Explicitly set the proxy settings to empty/zero to avoid relying on defaults that could potentially change
+            /// in the future.
+            aws_client_configuration.proxyHost = "";
+            aws_client_configuration.proxyUserName = "";
+            aws_client_configuration.proxyPassword = "";
+            aws_client_configuration.proxyPort = 0;
+
+            /// EC2MetadataService throttles by delaying the response so the service client should set a large read timeout.
+            /// EC2MetadataService delay is in order of seconds so it only make sense to retry after a couple of seconds.
+            aws_client_configuration.connectTimeoutMs = 1000;
+            aws_client_configuration.requestTimeoutMs = 1000;
+
+            aws_client_configuration.retryStrategy = std::make_shared<Aws::Client::DefaultRetryStrategy>(1, 1000);
+
+            auto ec2_metadata_client = InitEC2MetadataClient(aws_client_configuration);
+            auto config_loader = std::make_shared<AWSEC2InstanceProfileConfigLoader>(ec2_metadata_client, !use_insecure_imds_request);
+
+            AddProvider(std::make_shared<AWSInstanceProfileCredentialsProvider>(config_loader));
+            LOG_INFO(logger, "Added EC2 metadata service credentials provider to the provider chain.");
+        }
+    }
+
+    /// Quite verbose provider (argues if file with credentials doesn't exist) so iut's the last one
+    /// in chain.
+    AddProvider(std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>());
+}
+
+}
+
+#endif
--- a/src/IO/S3/Credentials.h
+++ b/src/IO/S3/Credentials.h
@ -0,0 +1,127 @@
+#pragma once
+
+#include "config.h"
+
+#if USE_AWS_S3
+#    include <aws/core/client/ClientConfiguration.h>
+#    include <aws/core/internal/AWSHttpResourceClient.h>
+#    include <aws/core/config/AWSProfileConfigLoader.h>
+#    include <aws/core/auth/AWSCredentialsProvider.h>
+#    include <aws/core/auth/AWSCredentialsProviderChain.h>
+
+#    include <Common/logger_useful.h>
+
+#    include <IO/S3/PocoHTTPClient.h>
+
+
+namespace DB::S3
+{
+
+class AWSEC2MetadataClient : public Aws::Internal::AWSHttpResourceClient
+{
+    static constexpr char EC2_SECURITY_CREDENTIALS_RESOURCE[] = "/latest/meta-data/iam/security-credentials";
+    static constexpr char EC2_IMDS_TOKEN_RESOURCE[] = "/latest/api/token";
+    static constexpr char EC2_IMDS_TOKEN_HEADER[] = "x-aws-ec2-metadata-token";
+    static constexpr char EC2_IMDS_TOKEN_TTL_DEFAULT_VALUE[] = "21600";
+    static constexpr char EC2_IMDS_TOKEN_TTL_HEADER[] = "x-aws-ec2-metadata-token-ttl-seconds";
+
+public:
+    /// See EC2MetadataClient.
+
+    explicit AWSEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration, const char * endpoint_);
+
+    AWSEC2MetadataClient& operator =(const AWSEC2MetadataClient & rhs) = delete;
+    AWSEC2MetadataClient(const AWSEC2MetadataClient & rhs) = delete;
+    AWSEC2MetadataClient& operator =(const AWSEC2MetadataClient && rhs) = delete;
+    AWSEC2MetadataClient(const AWSEC2MetadataClient && rhs) = delete;
+
+    ~AWSEC2MetadataClient() override = default;
+
+    using Aws::Internal::AWSHttpResourceClient::GetResource;
+
+    virtual Aws::String GetResource(const char * resource_path) const;
+    virtual Aws::String getDefaultCredentials() const;
+
+    static Aws::String awsComputeUserAgentString();
+
+    virtual Aws::String getDefaultCredentialsSecurely() const;
+
+    virtual Aws::String getCurrentRegion() const;
+
+private:
+    const Aws::String endpoint;
+    mutable std::recursive_mutex token_mutex;
+    mutable Aws::String token;
+    Poco::Logger * logger;
+};
+
+std::shared_ptr<AWSEC2MetadataClient> InitEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration);
+
+class AWSEC2InstanceProfileConfigLoader : public Aws::Config::AWSProfileConfigLoader
+{
+public:
+    explicit AWSEC2InstanceProfileConfigLoader(const std::shared_ptr<AWSEC2MetadataClient> & client_, bool use_secure_pull_);
+
+    ~AWSEC2InstanceProfileConfigLoader() override = default;
+
+protected:
+    bool LoadInternal() override;
+
+private:
+    std::shared_ptr<AWSEC2MetadataClient> client;
+    bool use_secure_pull;
+    Poco::Logger * logger;
+};
+
+class AWSInstanceProfileCredentialsProvider : public Aws::Auth::AWSCredentialsProvider
+{
+public:
+    /// See InstanceProfileCredentialsProvider.
+
+    explicit AWSInstanceProfileCredentialsProvider(const std::shared_ptr<AWSEC2InstanceProfileConfigLoader> & config_loader);
+
+    Aws::Auth::AWSCredentials GetAWSCredentials() override;
+protected:
+    void Reload() override;
+
+private:
+    void refreshIfExpired();
+
+    std::shared_ptr<AWSEC2InstanceProfileConfigLoader> ec2_metadata_config_loader;
+    Int64 load_frequency_ms;
+    Poco::Logger * logger;
+};
+
+class AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider : public Aws::Auth::AWSCredentialsProvider
+{
+    /// See STSAssumeRoleWebIdentityCredentialsProvider.
+
+public:
+    explicit AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider(DB::S3::PocoHTTPClientConfiguration & aws_client_configuration);
+
+    Aws::Auth::AWSCredentials GetAWSCredentials() override;
+protected:
+    void Reload() override;
+
+private:
+    void refreshIfExpired();
+
+    std::unique_ptr<Aws::Internal::STSCredentialsClient> client;
+    Aws::Auth::AWSCredentials credentials;
+    Aws::String role_arn;
+    Aws::String token_file;
+    Aws::String session_name;
+    Aws::String token;
+    bool initialized = false;
+    Poco::Logger * logger;
+};
+
+class S3CredentialsProviderChain : public Aws::Auth::AWSCredentialsProviderChain
+{
+public:
+    S3CredentialsProviderChain(const DB::S3::PocoHTTPClientConfiguration & configuration, const Aws::Auth::AWSCredentials & credentials, bool use_environment_credentials, bool use_insecure_imds_request);
+};
+
+}
+
+#endif
--- a/src/IO/S3/copyS3File.cpp
+++ b/src/IO/S3/copyS3File.cpp
@ -66,7 +66,17 @@ namespace
        {
        }

-        virtual ~UploadHelper() = default;
+        virtual ~UploadHelper()
+        {
+            try
+            {
+                waitForAllBackGroundTasks();
+            }
+            catch (...)
+            {
+                tryLogCurrentException(__PRETTY_FUNCTION__);
+            }
+        }

    protected:
        std::shared_ptr<const S3::Client> client_ptr;
--- a/src/IO/S3Common.cpp
+++ b/src/IO/S3Common.cpp
@ -12,25 +12,12 @@
 #    include <IO/HTTPHeaderEntries.h>
 #    include <Storages/StorageS3Settings.h>

-#    include <aws/core/Version.h>
-#    include <aws/core/auth/AWSCredentialsProvider.h>
-#    include <aws/core/auth/AWSCredentialsProviderChain.h>
-#    include <aws/core/auth/STSCredentialsProvider.h>
-#    include <aws/core/client/SpecifiedRetryableErrorsRetryStrategy.h>
-#    include <aws/core/platform/Environment.h>
-#    include <aws/core/platform/OSVersionInfo.h>
-#    include <aws/core/utils/json/JsonSerializer.h>
-#    include <aws/core/utils/logging/LogMacros.h>
-#    include <aws/core/utils/logging/LogSystemInterface.h>
-#    include <aws/core/utils/HashingUtils.h>
-#    include <aws/core/utils/UUID.h>
-#    include <aws/core/http/HttpClientFactory.h>
-
 #    include <IO/S3/PocoHTTPClientFactory.h>
 #    include <IO/S3/PocoHTTPClient.h>
 #    include <IO/S3/Client.h>
 #    include <IO/S3/URI.h>
 #    include <IO/S3/Requests.h>
+#    include <IO/S3/Credentials.h>
 #    include <Common/logger_useful.h>

 #    include <fstream>
@ -65,750 +52,11 @@ bool S3Exception::isRetryableError() const

 }

-namespace
-{
-
-const char * S3_LOGGER_TAG_NAMES[][2] = {
-    {"AWSClient", "AWSClient"},
-    {"AWSAuthV4Signer", "AWSClient (AWSAuthV4Signer)"},
-};
-
-const std::pair<DB::LogsLevel, Poco::Message::Priority> & convertLogLevel(Aws::Utils::Logging::LogLevel log_level)
-{
-    /// We map levels to our own logger 1 to 1 except WARN+ levels. In most cases we failover such errors with retries
-    /// and don't want to see them as Errors in our logs.
-    static const std::unordered_map<Aws::Utils::Logging::LogLevel, std::pair<DB::LogsLevel, Poco::Message::Priority>> mapping =
-    {
-        {Aws::Utils::Logging::LogLevel::Off, {DB::LogsLevel::none, Poco::Message::PRIO_INFORMATION}},
-        {Aws::Utils::Logging::LogLevel::Fatal, {DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION}},
-        {Aws::Utils::Logging::LogLevel::Error, {DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION}},
-        {Aws::Utils::Logging::LogLevel::Warn, {DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION}},
-        {Aws::Utils::Logging::LogLevel::Info, {DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION}},
-        {Aws::Utils::Logging::LogLevel::Debug, {DB::LogsLevel::debug, Poco::Message::PRIO_TEST}},
-        {Aws::Utils::Logging::LogLevel::Trace, {DB::LogsLevel::trace, Poco::Message::PRIO_TEST}},
-    };
-    return mapping.at(log_level);
-}
-
-class AWSLogger final : public Aws::Utils::Logging::LogSystemInterface
-{
-public:
-    explicit AWSLogger(bool enable_s3_requests_logging_)
-        :enable_s3_requests_logging(enable_s3_requests_logging_)
-    {
-        for (auto [tag, name] : S3_LOGGER_TAG_NAMES)
-            tag_loggers[tag] = &Poco::Logger::get(name);
-
-        default_logger = tag_loggers[S3_LOGGER_TAG_NAMES[0][0]];
-    }
-
-    ~AWSLogger() final = default;
-
-    Aws::Utils::Logging::LogLevel GetLogLevel() const final
-    {
-        if (enable_s3_requests_logging)
-            return Aws::Utils::Logging::LogLevel::Trace;
-        else
-            return Aws::Utils::Logging::LogLevel::Info;
-    }
-
-    void Log(Aws::Utils::Logging::LogLevel log_level, const char * tag, const char * format_str, ...) final // NOLINT
-    {
-        callLogImpl(log_level, tag, format_str); /// FIXME. Variadic arguments?
-    }
-
-    void LogStream(Aws::Utils::Logging::LogLevel log_level, const char * tag, const Aws::OStringStream & message_stream) final
-    {
-        callLogImpl(log_level, tag, message_stream.str().c_str());
-    }
-
-    void callLogImpl(Aws::Utils::Logging::LogLevel log_level, const char * tag, const char * message)
-    {
-        const auto & [level, prio] = convertLogLevel(log_level);
-        if (tag_loggers.contains(tag))
-        {
-            LOG_IMPL(tag_loggers[tag], level, prio, fmt::runtime(message));
-        }
-        else
-        {
-            LOG_IMPL(default_logger, level, prio, "{}: {}", tag, message);
-        }
-    }
-
-    void Flush() final {}
-
-private:
-    Poco::Logger * default_logger;
-    bool enable_s3_requests_logging;
-    std::unordered_map<String, Poco::Logger *> tag_loggers;
-};
-
-class AWSEC2MetadataClient : public Aws::Internal::AWSHttpResourceClient
-{
-    static constexpr char EC2_SECURITY_CREDENTIALS_RESOURCE[] = "/latest/meta-data/iam/security-credentials";
-    static constexpr char EC2_IMDS_TOKEN_RESOURCE[] = "/latest/api/token";
-    static constexpr char EC2_IMDS_TOKEN_HEADER[] = "x-aws-ec2-metadata-token";
-    static constexpr char EC2_IMDS_TOKEN_TTL_DEFAULT_VALUE[] = "21600";
-    static constexpr char EC2_IMDS_TOKEN_TTL_HEADER[] = "x-aws-ec2-metadata-token-ttl-seconds";
-
-public:
-    /// See EC2MetadataClient.
-
-    explicit AWSEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration, const char * endpoint_)
-        : Aws::Internal::AWSHttpResourceClient(client_configuration)
-        , endpoint(endpoint_)
-        , logger(&Poco::Logger::get("AWSEC2InstanceProfileConfigLoader"))
-    {
-    }
-
-    AWSEC2MetadataClient& operator =(const AWSEC2MetadataClient & rhs) = delete;
-    AWSEC2MetadataClient(const AWSEC2MetadataClient & rhs) = delete;
-    AWSEC2MetadataClient& operator =(const AWSEC2MetadataClient && rhs) = delete;
-    AWSEC2MetadataClient(const AWSEC2MetadataClient && rhs) = delete;
-
-    ~AWSEC2MetadataClient() override = default;
-
-    using Aws::Internal::AWSHttpResourceClient::GetResource;
-
-    virtual Aws::String GetResource(const char * resource_path) const
-    {
-        return GetResource(endpoint.c_str(), resource_path, nullptr/*authToken*/);
-    }
-
-    virtual Aws::String getDefaultCredentials() const
-    {
-        String credentials_string;
-        {
-            std::lock_guard locker(token_mutex);
-
-            LOG_TRACE(logger, "Getting default credentials for ec2 instance from {}", endpoint);
-            auto result = GetResourceWithAWSWebServiceResult(endpoint.c_str(), EC2_SECURITY_CREDENTIALS_RESOURCE, nullptr);
-            credentials_string = result.GetPayload();
-            if (result.GetResponseCode() == Aws::Http::HttpResponseCode::UNAUTHORIZED)
-            {
-                return {};
-            }
-        }
-
-        String trimmed_credentials_string = Aws::Utils::StringUtils::Trim(credentials_string.c_str());
-        if (trimmed_credentials_string.empty())
-            return {};
-
-        std::vector<String> security_credentials = Aws::Utils::StringUtils::Split(trimmed_credentials_string, '\n');
-
-        LOG_DEBUG(logger, "Calling EC2MetadataService resource, {} returned credential string {}.",
-                EC2_SECURITY_CREDENTIALS_RESOURCE, trimmed_credentials_string);
-
-        if (security_credentials.empty())
-        {
-            LOG_WARNING(logger, "Initial call to EC2MetadataService to get credentials failed.");
-            return {};
-        }
-
-        Aws::StringStream ss;
-        ss << EC2_SECURITY_CREDENTIALS_RESOURCE << "/" << security_credentials[0];
-        LOG_DEBUG(logger, "Calling EC2MetadataService resource {}.", ss.str());
-        return GetResource(ss.str().c_str());
-    }
-
-    static Aws::String awsComputeUserAgentString()
-    {
-        Aws::StringStream ss;
-        ss << "aws-sdk-cpp/" << Aws::Version::GetVersionString() << " " << Aws::OSVersionInfo::ComputeOSVersionString()
-                << " " << Aws::Version::GetCompilerVersionString();
-        return ss.str();
-    }
-
-    virtual Aws::String getDefaultCredentialsSecurely() const
-    {
-        String user_agent_string = awsComputeUserAgentString();
-        String new_token;
-
-        {
-            std::lock_guard locker(token_mutex);
-
-            Aws::StringStream ss;
-            ss << endpoint << EC2_IMDS_TOKEN_RESOURCE;
-            std::shared_ptr<Aws::Http::HttpRequest> token_request(Aws::Http::CreateHttpRequest(ss.str(), Aws::Http::HttpMethod::HTTP_PUT,
-                                                                        Aws::Utils::Stream::DefaultResponseStreamFactoryMethod));
-            token_request->SetHeaderValue(EC2_IMDS_TOKEN_TTL_HEADER, EC2_IMDS_TOKEN_TTL_DEFAULT_VALUE);
-            token_request->SetUserAgent(user_agent_string);
-            LOG_TRACE(logger, "Calling EC2MetadataService to get token.");
-            auto result = GetResourceWithAWSWebServiceResult(token_request);
-            const String & token_string = result.GetPayload();
-            new_token = Aws::Utils::StringUtils::Trim(token_string.c_str());
-
-            if (result.GetResponseCode() == Aws::Http::HttpResponseCode::BAD_REQUEST)
-            {
-                return {};
-            }
-            else if (result.GetResponseCode() != Aws::Http::HttpResponseCode::OK || new_token.empty())
-            {
-                LOG_TRACE(logger, "Calling EC2MetadataService to get token failed, falling back to less secure way.");
-                return getDefaultCredentials();
-            }
-            token = new_token;
-        }
-
-        String url = endpoint + EC2_SECURITY_CREDENTIALS_RESOURCE;
-        std::shared_ptr<Aws::Http::HttpRequest> profile_request(Aws::Http::CreateHttpRequest(url,
-                Aws::Http::HttpMethod::HTTP_GET,
-                Aws::Utils::Stream::DefaultResponseStreamFactoryMethod));
-        profile_request->SetHeaderValue(EC2_IMDS_TOKEN_HEADER, new_token);
-        profile_request->SetUserAgent(user_agent_string);
-        String profile_string = GetResourceWithAWSWebServiceResult(profile_request).GetPayload();
-
-        String trimmed_profile_string = Aws::Utils::StringUtils::Trim(profile_string.c_str());
-        std::vector<String> security_credentials = Aws::Utils::StringUtils::Split(trimmed_profile_string, '\n');
-
-        LOG_DEBUG(logger, "Calling EC2MetadataService resource, {} with token returned profile string {}.",
-                EC2_SECURITY_CREDENTIALS_RESOURCE, trimmed_profile_string);
-
-        if (security_credentials.empty())
-        {
-            LOG_WARNING(logger, "Calling EC2Metadataservice to get profiles failed.");
-            return {};
-        }
-
-        Aws::StringStream ss;
-        ss << endpoint << EC2_SECURITY_CREDENTIALS_RESOURCE << "/" << security_credentials[0];
-        std::shared_ptr<Aws::Http::HttpRequest> credentials_request(Aws::Http::CreateHttpRequest(ss.str(),
-                Aws::Http::HttpMethod::HTTP_GET,
-                Aws::Utils::Stream::DefaultResponseStreamFactoryMethod));
-        credentials_request->SetHeaderValue(EC2_IMDS_TOKEN_HEADER, new_token);
-        credentials_request->SetUserAgent(user_agent_string);
-        LOG_DEBUG(logger, "Calling EC2MetadataService resource {} with token.", ss.str());
-        return GetResourceWithAWSWebServiceResult(credentials_request).GetPayload();
-    }
-
-    virtual Aws::String getCurrentRegion() const
-    {
-        return Aws::Region::AWS_GLOBAL;
-    }
-
-private:
-    const Aws::String endpoint;
-    mutable std::recursive_mutex token_mutex;
-    mutable Aws::String token;
-    Poco::Logger * logger;
-};
-
-std::shared_ptr<AWSEC2MetadataClient> InitEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration)
-{
-    Aws::String ec2_metadata_service_endpoint = Aws::Environment::GetEnv("AWS_EC2_METADATA_SERVICE_ENDPOINT");
-    auto * logger = &Poco::Logger::get("AWSEC2InstanceProfileConfigLoader");
-    if (ec2_metadata_service_endpoint.empty())
-    {
-        Aws::String ec2_metadata_service_endpoint_mode = Aws::Environment::GetEnv("AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE");
-        if (ec2_metadata_service_endpoint_mode.length() == 0)
-        {
-            ec2_metadata_service_endpoint = "http://169.254.169.254"; //default to IPv4 default endpoint
-        }
-        else
-        {
-            if (ec2_metadata_service_endpoint_mode.length() == 4)
-            {
-                if (Aws::Utils::StringUtils::CaselessCompare(ec2_metadata_service_endpoint_mode.c_str(), "ipv4"))
-                {
-                    ec2_metadata_service_endpoint = "http://169.254.169.254"; //default to IPv4 default endpoint
-                }
-                else if (Aws::Utils::StringUtils::CaselessCompare(ec2_metadata_service_endpoint_mode.c_str(), "ipv6"))
-                {
-                    ec2_metadata_service_endpoint = "http://[fd00:ec2::254]";
-                }
-                else
-                {
-                    LOG_ERROR(logger, "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE can only be set to ipv4 or ipv6, received: {}", ec2_metadata_service_endpoint_mode);
-                }
-            }
-            else
-            {
-                LOG_ERROR(logger, "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE can only be set to ipv4 or ipv6, received: {}", ec2_metadata_service_endpoint_mode);
-            }
-        }
-    }
-    LOG_INFO(logger, "Using IMDS endpoint: {}", ec2_metadata_service_endpoint);
-    return std::make_shared<AWSEC2MetadataClient>(client_configuration, ec2_metadata_service_endpoint.c_str());
-}
-
-class AWSEC2InstanceProfileConfigLoader : public Aws::Config::AWSProfileConfigLoader
-{
-public:
-    explicit AWSEC2InstanceProfileConfigLoader(const std::shared_ptr<AWSEC2MetadataClient> & client_, bool use_secure_pull_)
-        : client(client_)
-        , use_secure_pull(use_secure_pull_)
-        , logger(&Poco::Logger::get("AWSEC2InstanceProfileConfigLoader"))
-    {
-    }
-
-    ~AWSEC2InstanceProfileConfigLoader() override = default;
-
-protected:
-    bool LoadInternal() override
-    {
-        auto credentials_str = use_secure_pull ? client->getDefaultCredentialsSecurely() : client->getDefaultCredentials();
-
-        /// See EC2InstanceProfileConfigLoader.
-        if (credentials_str.empty())
-            return false;
-
-        Aws::Utils::Json::JsonValue credentials_doc(credentials_str);
-        if (!credentials_doc.WasParseSuccessful())
-        {
-            LOG_ERROR(logger, "Failed to parse output from EC2MetadataService.");
-            return false;
-        }
-        String access_key, secret_key, token;
-
-        auto credentials_view = credentials_doc.View();
-        access_key = credentials_view.GetString("AccessKeyId");
-        LOG_TRACE(logger, "Successfully pulled credentials from EC2MetadataService with access key.");
-
-        secret_key = credentials_view.GetString("SecretAccessKey");
-        token = credentials_view.GetString("Token");
-
-        auto region = client->getCurrentRegion();
-
-        Aws::Config::Profile profile;
-        profile.SetCredentials(Aws::Auth::AWSCredentials(access_key, secret_key, token));
-        profile.SetRegion(region);
-        profile.SetName(Aws::Config::INSTANCE_PROFILE_KEY);
-
-        m_profiles[Aws::Config::INSTANCE_PROFILE_KEY] = profile;
-
-        return true;
-    }
-
-private:
-    std::shared_ptr<AWSEC2MetadataClient> client;
-    bool use_secure_pull;
-    Poco::Logger * logger;
-};
-
-class AWSInstanceProfileCredentialsProvider : public Aws::Auth::AWSCredentialsProvider
-{
-public:
-    /// See InstanceProfileCredentialsProvider.
-
-    explicit AWSInstanceProfileCredentialsProvider(const std::shared_ptr<AWSEC2InstanceProfileConfigLoader> & config_loader)
-        : ec2_metadata_config_loader(config_loader)
-        , load_frequency_ms(Aws::Auth::REFRESH_THRESHOLD)
-        , logger(&Poco::Logger::get("AWSInstanceProfileCredentialsProvider"))
-    {
-        LOG_INFO(logger, "Creating Instance with injected EC2MetadataClient and refresh rate.");
-    }
-
-    Aws::Auth::AWSCredentials GetAWSCredentials() override
-    {
-        refreshIfExpired();
-        Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock);
-        auto profile_it = ec2_metadata_config_loader->GetProfiles().find(Aws::Config::INSTANCE_PROFILE_KEY);
-
-        if (profile_it != ec2_metadata_config_loader->GetProfiles().end())
-        {
-            return profile_it->second.GetCredentials();
-        }
-
-        return Aws::Auth::AWSCredentials();
-    }
-
-protected:
-    void Reload() override
-    {
-        LOG_INFO(logger, "Credentials have expired attempting to repull from EC2 Metadata Service.");
-        ec2_metadata_config_loader->Load();
-        AWSCredentialsProvider::Reload();
-    }
-
-private:
-    void refreshIfExpired()
-    {
-        LOG_DEBUG(logger, "Checking if latest credential pull has expired.");
-        Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock);
-        if (!IsTimeToRefresh(load_frequency_ms))
-        {
-            return;
-        }
-
-        guard.UpgradeToWriterLock();
-        if (!IsTimeToRefresh(load_frequency_ms)) // double-checked lock to avoid refreshing twice
-        {
-            return;
-        }
-        Reload();
-    }
-
-    std::shared_ptr<AWSEC2InstanceProfileConfigLoader> ec2_metadata_config_loader;
-    Int64 load_frequency_ms;
-    Poco::Logger * logger;
-};
-
-class AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider : public Aws::Auth::AWSCredentialsProvider
-{
-    /// See STSAssumeRoleWebIdentityCredentialsProvider.
-
-public:
-    explicit AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider(DB::S3::PocoHTTPClientConfiguration & aws_client_configuration)
-        : logger(&Poco::Logger::get("AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider"))
-    {
-        // check environment variables
-        String tmp_region = Aws::Environment::GetEnv("AWS_DEFAULT_REGION");
-        role_arn = Aws::Environment::GetEnv("AWS_ROLE_ARN");
-        token_file = Aws::Environment::GetEnv("AWS_WEB_IDENTITY_TOKEN_FILE");
-        session_name = Aws::Environment::GetEnv("AWS_ROLE_SESSION_NAME");
-
-        // check profile_config if either m_roleArn or m_tokenFile is not loaded from environment variable
-        // region source is not enforced, but we need it to construct sts endpoint, if we can't find from environment, we should check if it's set in config file.
-        if (role_arn.empty() || token_file.empty() || tmp_region.empty())
-        {
-            auto profile = Aws::Config::GetCachedConfigProfile(Aws::Auth::GetConfigProfileName());
-            if (tmp_region.empty())
-            {
-                tmp_region = profile.GetRegion();
-            }
-            // If either of these two were not found from environment, use whatever found for all three in config file
-            if (role_arn.empty() || token_file.empty())
-            {
-                role_arn = profile.GetRoleArn();
-                token_file = profile.GetValue("web_identity_token_file");
-                session_name = profile.GetValue("role_session_name");
-            }
-        }
-
-        if (token_file.empty())
-        {
-            LOG_WARNING(logger, "Token file must be specified to use STS AssumeRole web identity creds provider.");
-            return; // No need to do further constructing
-        }
-        else
-        {
-            LOG_DEBUG(logger, "Resolved token_file from profile_config or environment variable to be {}", token_file);
-        }
-
-        if (role_arn.empty())
-        {
-            LOG_WARNING(logger, "RoleArn must be specified to use STS AssumeRole web identity creds provider.");
-            return; // No need to do further constructing
-        }
-        else
-        {
-            LOG_DEBUG(logger, "Resolved role_arn from profile_config or environment variable to be {}", role_arn);
-        }
-
-        if (tmp_region.empty())
-        {
-            tmp_region = Aws::Region::US_EAST_1;
-        }
-        else
-        {
-            LOG_DEBUG(logger, "Resolved region from profile_config or environment variable to be {}", tmp_region);
-        }
-
-        if (session_name.empty())
-        {
-            session_name = Aws::Utils::UUID::RandomUUID();
-        }
-        else
-        {
-            LOG_DEBUG(logger, "Resolved session_name from profile_config or environment variable to be {}", session_name);
-        }
-
-        aws_client_configuration.scheme = Aws::Http::Scheme::HTTPS;
-        aws_client_configuration.region = tmp_region;
-
-        std::vector<String> retryable_errors;
-        retryable_errors.push_back("IDPCommunicationError");
-        retryable_errors.push_back("InvalidIdentityToken");
-
-        aws_client_configuration.retryStrategy = std::make_shared<Aws::Client::SpecifiedRetryableErrorsRetryStrategy>(
-            retryable_errors, /* maxRetries = */3);
-
-        client = std::make_unique<Aws::Internal::STSCredentialsClient>(aws_client_configuration);
-        initialized = true;
-        LOG_INFO(logger, "Creating STS AssumeRole with web identity creds provider.");
-    }
-
-    Aws::Auth::AWSCredentials GetAWSCredentials() override
-    {
-        // A valid client means required information like role arn and token file were constructed correctly.
-        // We can use this provider to load creds, otherwise, we can just return empty creds.
-        if (!initialized)
-        {
-            return Aws::Auth::AWSCredentials();
-        }
-        refreshIfExpired();
-        Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock);
-        return credentials;
-    }
-
-protected:
-    void Reload() override
-    {
-        LOG_INFO(logger, "Credentials have expired, attempting to renew from STS.");
-
-        std::ifstream token_stream(token_file.data());
-        if (token_stream)
-        {
-            String token_string((std::istreambuf_iterator<char>(token_stream)), std::istreambuf_iterator<char>());
-            token = token_string;
-        }
-        else
-        {
-            LOG_INFO(logger, "Can't open token file: {}", token_file);
-            return;
-        }
-        Aws::Internal::STSCredentialsClient::STSAssumeRoleWithWebIdentityRequest request{session_name, role_arn, token};
-
-        auto result = client->GetAssumeRoleWithWebIdentityCredentials(request);
-        LOG_TRACE(logger, "Successfully retrieved credentials.");
-        credentials = result.creds;
-    }
-
-private:
-    void refreshIfExpired()
-    {
-        Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock);
-        if (!credentials.IsExpiredOrEmpty())
-        {
-            return;
-        }
-
-        guard.UpgradeToWriterLock();
-        if (!credentials.IsExpiredOrEmpty()) // double-checked lock to avoid refreshing twice
-        {
-            return;
-        }
-
-        Reload();
-    }
-
-    std::unique_ptr<Aws::Internal::STSCredentialsClient> client;
-    Aws::Auth::AWSCredentials credentials;
-    Aws::String role_arn;
-    Aws::String token_file;
-    Aws::String session_name;
-    Aws::String token;
-    bool initialized = false;
-    Poco::Logger * logger;
-};
-
-class S3CredentialsProviderChain : public Aws::Auth::AWSCredentialsProviderChain
-{
-public:
-    S3CredentialsProviderChain(const DB::S3::PocoHTTPClientConfiguration & configuration, const Aws::Auth::AWSCredentials & credentials, bool use_environment_credentials, bool use_insecure_imds_request)
-    {
-        auto * logger = &Poco::Logger::get("S3CredentialsProviderChain");
-
-        /// add explicit credentials to the front of the chain
-        /// because it's manually defined by the user
-        if (!credentials.IsEmpty())
-        {
-            AddProvider(std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>(credentials));
-            return;
-        }
-
-        if (use_environment_credentials)
-        {
-            static const char AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI[] = "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI";
-            static const char AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI[] = "AWS_CONTAINER_CREDENTIALS_FULL_URI";
-            static const char AWS_ECS_CONTAINER_AUTHORIZATION_TOKEN[] = "AWS_CONTAINER_AUTHORIZATION_TOKEN";
-            static const char AWS_EC2_METADATA_DISABLED[] = "AWS_EC2_METADATA_DISABLED";
-
-            /// The only difference from DefaultAWSCredentialsProviderChain::DefaultAWSCredentialsProviderChain()
-            /// is that this chain uses custom ClientConfiguration. Also we removed process provider because it's useless in our case.
-            ///
-            /// AWS API tries credentials providers one by one. Some of providers (like ProfileConfigFileAWSCredentialsProvider) can be
-            /// quite verbose even if nobody configured them. So we use our provider first and only after it use default providers.
-            {
-                DB::S3::PocoHTTPClientConfiguration aws_client_configuration = DB::S3::ClientFactory::instance().createClientConfiguration(
-                    configuration.region,
-                    configuration.remote_host_filter,
-                    configuration.s3_max_redirects,
-                    configuration.enable_s3_requests_logging,
-                    configuration.for_disk_s3,
-                    configuration.get_request_throttler,
-                    configuration.put_request_throttler);
-                AddProvider(std::make_shared<AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider>(aws_client_configuration));
-            }
-
-            AddProvider(std::make_shared<Aws::Auth::EnvironmentAWSCredentialsProvider>());
-
-
-            /// ECS TaskRole Credentials only available when ENVIRONMENT VARIABLE is set.
-            const auto relative_uri = Aws::Environment::GetEnv(AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI);
-            LOG_DEBUG(logger, "The environment variable value {} is {}", AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI,
-                    relative_uri);
-
-            const auto absolute_uri = Aws::Environment::GetEnv(AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI);
-            LOG_DEBUG(logger, "The environment variable value {} is {}", AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI,
-                    absolute_uri);
-
-            const auto ec2_metadata_disabled = Aws::Environment::GetEnv(AWS_EC2_METADATA_DISABLED);
-            LOG_DEBUG(logger, "The environment variable value {} is {}", AWS_EC2_METADATA_DISABLED,
-                    ec2_metadata_disabled);
-
-            if (!relative_uri.empty())
-            {
-                AddProvider(std::make_shared<Aws::Auth::TaskRoleCredentialsProvider>(relative_uri.c_str()));
-                LOG_INFO(logger, "Added ECS metadata service credentials provider with relative path: [{}] to the provider chain.",
-                        relative_uri);
-            }
-            else if (!absolute_uri.empty())
-            {
-                const auto token = Aws::Environment::GetEnv(AWS_ECS_CONTAINER_AUTHORIZATION_TOKEN);
-                AddProvider(std::make_shared<Aws::Auth::TaskRoleCredentialsProvider>(absolute_uri.c_str(), token.c_str()));
-
-                /// DO NOT log the value of the authorization token for security purposes.
-                LOG_INFO(logger, "Added ECS credentials provider with URI: [{}] to the provider chain with a{} authorization token.",
-                        absolute_uri, token.empty() ? "n empty" : " non-empty");
-            }
-            else if (Aws::Utils::StringUtils::ToLower(ec2_metadata_disabled.c_str()) != "true")
-            {
-                DB::S3::PocoHTTPClientConfiguration aws_client_configuration = DB::S3::ClientFactory::instance().createClientConfiguration(
-                    configuration.region,
-                    configuration.remote_host_filter,
-                    configuration.s3_max_redirects,
-                    configuration.enable_s3_requests_logging,
-                    configuration.for_disk_s3,
-                    configuration.get_request_throttler,
-                    configuration.put_request_throttler);
-
-                /// See MakeDefaultHttpResourceClientConfiguration().
-                /// This is part of EC2 metadata client, but unfortunately it can't be accessed from outside
-                /// of contrib/aws/aws-cpp-sdk-core/source/internal/AWSHttpResourceClient.cpp
-                aws_client_configuration.maxConnections = 2;
-                aws_client_configuration.scheme = Aws::Http::Scheme::HTTP;
-
-                /// Explicitly set the proxy settings to empty/zero to avoid relying on defaults that could potentially change
-                /// in the future.
-                aws_client_configuration.proxyHost = "";
-                aws_client_configuration.proxyUserName = "";
-                aws_client_configuration.proxyPassword = "";
-                aws_client_configuration.proxyPort = 0;
-
-                /// EC2MetadataService throttles by delaying the response so the service client should set a large read timeout.
-                /// EC2MetadataService delay is in order of seconds so it only make sense to retry after a couple of seconds.
-                aws_client_configuration.connectTimeoutMs = 1000;
-                aws_client_configuration.requestTimeoutMs = 1000;
-
-                aws_client_configuration.retryStrategy = std::make_shared<Aws::Client::DefaultRetryStrategy>(1, 1000);
-
-                auto ec2_metadata_client = InitEC2MetadataClient(aws_client_configuration);
-                auto config_loader = std::make_shared<AWSEC2InstanceProfileConfigLoader>(ec2_metadata_client, !use_insecure_imds_request);
-
-                AddProvider(std::make_shared<AWSInstanceProfileCredentialsProvider>(config_loader));
-                LOG_INFO(logger, "Added EC2 metadata service credentials provider to the provider chain.");
-            }
-        }
-
-        /// Quite verbose provider (argues if file with credentials doesn't exist) so iut's the last one
-        /// in chain.
-        AddProvider(std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>());
-    }
-};
-
-}
-
-
-namespace DB
-{
-namespace ErrorCodes
+namespace DB::ErrorCodes
 {
    extern const int S3_ERROR;
 }

-namespace S3
-{
-    ClientFactory::ClientFactory()
-    {
-        aws_options = Aws::SDKOptions{};
-        Aws::InitAPI(aws_options);
-        Aws::Utils::Logging::InitializeAWSLogging(std::make_shared<AWSLogger>(false));
-        Aws::Http::SetHttpClientFactory(std::make_shared<PocoHTTPClientFactory>());
-    }
-
-    ClientFactory::~ClientFactory()
-    {
-        Aws::Utils::Logging::ShutdownAWSLogging();
-        Aws::ShutdownAPI(aws_options);
-    }
-
-    ClientFactory & ClientFactory::instance()
-    {
-        static ClientFactory ret;
-        return ret;
-    }
-
-    std::unique_ptr<S3::Client> ClientFactory::create( // NOLINT
-        const PocoHTTPClientConfiguration & cfg_,
-        bool is_virtual_hosted_style,
-        const String & access_key_id,
-        const String & secret_access_key,
-        const String & server_side_encryption_customer_key_base64,
-        HTTPHeaderEntries headers,
-        bool use_environment_credentials,
-        bool use_insecure_imds_request)
-    {
-        PocoHTTPClientConfiguration client_configuration = cfg_;
-        client_configuration.updateSchemeAndRegion();
-
-        if (!server_side_encryption_customer_key_base64.empty())
-        {
-            /// See Client::GeneratePresignedUrlWithSSEC().
-
-            headers.push_back({Aws::S3::SSEHeaders::SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM,
-                Aws::S3::Model::ServerSideEncryptionMapper::GetNameForServerSideEncryption(Aws::S3::Model::ServerSideEncryption::AES256)});
-
-            headers.push_back({Aws::S3::SSEHeaders::SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
-                server_side_encryption_customer_key_base64});
-
-            Aws::Utils::ByteBuffer buffer = Aws::Utils::HashingUtils::Base64Decode(server_side_encryption_customer_key_base64);
-            String str_buffer(reinterpret_cast<char *>(buffer.GetUnderlyingData()), buffer.GetLength());
-            headers.push_back({Aws::S3::SSEHeaders::SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
-                Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateMD5(str_buffer))});
-        }
-
-        client_configuration.extra_headers = std::move(headers);
-
-        Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key);
-        auto credentials_provider = std::make_shared<S3CredentialsProviderChain>(
-                client_configuration,
-                std::move(credentials),
-                use_environment_credentials,
-                use_insecure_imds_request);
-
-        client_configuration.retryStrategy = std::make_shared<Client::RetryStrategy>(std::move(client_configuration.retryStrategy));
-        return Client::create(
-            client_configuration.s3_max_redirects,
-            std::move(credentials_provider),
-            std::move(client_configuration), // Client configuration.
-            Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
-            is_virtual_hosted_style || client_configuration.endpointOverride.empty() /// Use virtual addressing if endpoint is not specified.
-        );
-    }
-
-    PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT
-        const String & force_region,
-        const RemoteHostFilter & remote_host_filter,
-        unsigned int s3_max_redirects,
-        bool enable_s3_requests_logging,
-        bool for_disk_s3,
-        const ThrottlerPtr & get_request_throttler,
-        const ThrottlerPtr & put_request_throttler)
-    {
-        return PocoHTTPClientConfiguration(
-            force_region,
-            remote_host_filter,
-            s3_max_redirects,
-            enable_s3_requests_logging,
-            for_disk_s3,
-            get_request_throttler,
-            put_request_throttler);
-    }
-}
-
-}
-
 #endif

 namespace DB
--- a/src/IO/S3Common.h
+++ b/src/IO/S3Common.h
@ -61,43 +61,6 @@ private:
 };
 }

-namespace DB::S3
-{
-
-class ClientFactory
-{
-public:
-    ~ClientFactory();
-
-    static ClientFactory & instance();
-
-    std::unique_ptr<S3::Client> create(
-        const PocoHTTPClientConfiguration & cfg,
-        bool is_virtual_hosted_style,
-        const String & access_key_id,
-        const String & secret_access_key,
-        const String & server_side_encryption_customer_key_base64,
-        HTTPHeaderEntries headers,
-        bool use_environment_credentials,
-        bool use_insecure_imds_request);
-
-    PocoHTTPClientConfiguration createClientConfiguration(
-        const String & force_region,
-        const RemoteHostFilter & remote_host_filter,
-        unsigned int s3_max_redirects,
-        bool enable_s3_requests_logging,
-        bool for_disk_s3,
-        const ThrottlerPtr & get_request_throttler,
-        const ThrottlerPtr & put_request_throttler);
-
-private:
-    ClientFactory();
-
-    Aws::SDKOptions aws_options;
-    std::atomic<bool> s3_requests_logging_enabled;
-};
-
-}
 #endif

 namespace Poco::Util
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@ -448,10 +448,6 @@ InterpreterSelectQuery::InterpreterSelectQuery(
        }
    }

-    /// FIXME: Memory bound aggregation may cause another reading algorithm to be used on remote replicas
-    if (settings.allow_experimental_parallel_reading_from_replicas && settings.enable_memory_bound_merging_of_aggregation_results)
-        context->setSetting("enable_memory_bound_merging_of_aggregation_results", false);
-
    if (joined_tables.tablesCount() > 1 && settings.allow_experimental_parallel_reading_from_replicas)
    {
        LOG_WARNING(log, "Joins are not supported with parallel replicas. Query will be executed without using them.");
@ -1534,7 +1530,25 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P

                        auto join_kind = table_join.kind();
                        bool kind_allows_filtering = isInner(join_kind) || isLeft(join_kind) || isRight(join_kind);
-                        if (settings.max_rows_in_set_to_optimize_join > 0 && kind_allows_filtering)
+
+                        auto has_non_const = [](const Block & block, const auto & keys)
+                        {
+                            for (const auto & key : keys)
+                            {
+                                const auto & column = block.getByName(key).column;
+                                if (column && !isColumnConst(*column))
+                                    return true;
+                            }
+                            return false;
+                        };
+                        /// This optimization relies on the sorting that should buffer the whole stream before emitting any rows.
+                        /// It doesn't hold such a guarantee for streams with const keys.
+                        /// Note: it's also doesn't work with the read-in-order optimization.
+                        /// No checks here because read in order is not applied if we have `CreateSetAndFilterOnTheFlyStep` in the pipeline between the reading and sorting steps.
+                        bool has_non_const_keys = has_non_const(query_plan.getCurrentDataStream().header, join_clause.key_names_left)
+                            && has_non_const(joined_plan->getCurrentDataStream().header, join_clause.key_names_right);
+
+                        if (settings.max_rows_in_set_to_optimize_join > 0 && kind_allows_filtering && has_non_const_keys)
                        {
                            auto * left_set = add_create_set(query_plan, join_clause.key_names_left, JoinTableSide::Left);
                            auto * right_set = add_create_set(*joined_plan, join_clause.key_names_right, JoinTableSide::Right);
@ -2502,24 +2516,8 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac

    if (!group_by_info && settings.force_aggregation_in_order)
    {
-        /// Not the most optimal implementation here, but this branch handles very marginal case.
-
        group_by_sort_description = getSortDescriptionFromGroupBy(getSelectQuery());
-
-        auto sorting_step = std::make_unique<SortingStep>(
-            query_plan.getCurrentDataStream(),
-            group_by_sort_description,
-            0 /* LIMIT */,
-            SortingStep::Settings(*context),
-            settings.optimize_sorting_by_input_stream_properties);
-        sorting_step->setStepDescription("Enforced sorting for aggregation in order");
-
-        query_plan.addStep(std::move(sorting_step));
-
-        group_by_info = std::make_shared<InputOrderInfo>(
-            group_by_sort_description, group_by_sort_description.size(), 1 /* direction */, 0 /* limit */);
-
-        sort_description_for_merging = group_by_info->sort_description_for_merging;
+        sort_description_for_merging = group_by_sort_description;
    }

    auto merge_threads = max_streams;
@ -2546,7 +2544,8 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac
        std::move(sort_description_for_merging),
        std::move(group_by_sort_description),
        should_produce_results_in_order_of_bucket_number,
-        settings.enable_memory_bound_merging_of_aggregation_results);
+        settings.enable_memory_bound_merging_of_aggregation_results,
+        !group_by_info && settings.force_aggregation_in_order);
    query_plan.addStep(std::move(aggregating_step));
 }

--- a/src/Interpreters/RewriteArrayExistsFunctionVisitor.cpp
+++ b/src/Interpreters/RewriteArrayExistsFunctionVisitor.cpp
@ -0,0 +1,79 @@
+#include <Interpreters/RewriteArrayExistsFunctionVisitor.h>
+#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTIdentifier.h>
+#include <Parsers/ASTLiteral.h>
+
+namespace DB
+{
+void RewriteArrayExistsFunctionMatcher::visit(ASTPtr & ast, Data & data)
+{
+    if (auto * func = ast->as<ASTFunction>())
+    {
+        if (func->is_window_function)
+            return;
+
+        visit(*func, ast, data);
+    }
+}
+
+void RewriteArrayExistsFunctionMatcher::visit(const ASTFunction & func, ASTPtr & ast, Data &)
+{
+    if (func.name != "arrayExists" || !func.arguments)
+        return;
+
+    auto & array_exists_arguments = func.arguments->children;
+    if (array_exists_arguments.size() != 2)
+        return;
+
+    /// lambda function must be like: x -> x = elem
+    const auto * lambda_func = array_exists_arguments[0]->as<ASTFunction>();
+    if (!lambda_func || !lambda_func->is_lambda_function)
+        return;
+
+    const auto & lambda_func_arguments = lambda_func->arguments->children;
+    if (lambda_func_arguments.size() != 2)
+        return;
+
+    const auto * tuple_func = lambda_func_arguments[0]->as<ASTFunction>();
+    if (!tuple_func || tuple_func->name != "tuple")
+        return;
+
+    const auto & tuple_arguments = tuple_func->arguments->children;
+    if (tuple_arguments.size() != 1)
+        return;
+
+    const auto * id = tuple_arguments[0]->as<ASTIdentifier>();
+    if (!id)
+        return;
+
+    const auto * filter_func = lambda_func_arguments[1]->as<ASTFunction>();
+    if (!filter_func || filter_func->name != "equals")
+        return;
+
+    auto & filter_arguments = filter_func->arguments->children;
+    if (filter_arguments.size() != 2)
+        return;
+
+    const ASTIdentifier * filter_id = nullptr;
+    if ((filter_id = filter_arguments[0]->as<ASTIdentifier>()) && filter_arguments[1]->as<ASTLiteral>()
+        && filter_id->full_name == id->full_name)
+    {
+        /// arrayExists(x -> x = elem, arr) -> has(arr, elem)
+        auto new_func = makeASTFunction("has", std::move(array_exists_arguments[1]), std::move(filter_arguments[1]));
+        new_func->setAlias(func.alias);
+        ast = std::move(new_func);
+        return;
+    }
+    else if (
+        (filter_id = filter_arguments[1]->as<ASTIdentifier>()) && filter_arguments[0]->as<ASTLiteral>()
+        && filter_id->full_name == id->full_name)
+    {
+        /// arrayExists(x -> elem = x, arr) -> has(arr, elem)
+        auto new_func = makeASTFunction("has", std::move(array_exists_arguments[1]), std::move(filter_arguments[0]));
+        new_func->setAlias(func.alias);
+        ast = std::move(new_func);
+        return;
+    }
+}
+
+}
--- a/src/Interpreters/RewriteArrayExistsFunctionVisitor.h
+++ b/src/Interpreters/RewriteArrayExistsFunctionVisitor.h
@ -0,0 +1,25 @@
+#pragma once
+
+#include <Interpreters/InDepthNodeVisitor.h>
+#include <Parsers/IAST.h>
+
+namespace DB
+{
+class ASTFunction;
+
+/// Rewrite possible 'arrayExists(func, arr)' to 'has(arr, elem)' to improve performance
+/// arrayExists(x -> x = 1, arr) -> has(arr, 1)
+class RewriteArrayExistsFunctionMatcher
+{
+public:
+    struct Data
+    {
+    };
+
+    static void visit(ASTPtr & ast, Data &);
+    static void visit(const ASTFunction &, ASTPtr & ast, Data &);
+    static bool needChildVisit(const ASTPtr &, const ASTPtr &) { return true; }
+};
+
+using RewriteArrayExistsFunctionVisitor = InDepthNodeVisitor<RewriteArrayExistsFunctionMatcher, false>;
+}
--- a/src/Interpreters/ServerAsynchronousMetrics.cpp
+++ b/src/Interpreters/ServerAsynchronousMetrics.cpp
@ -245,7 +245,7 @@ void ServerAsynchronousMetrics::updateImpl(AsynchronousMetricValues & new_values
                    calculateMax(max_part_count_for_partition, table_merge_tree->getMaxPartsCountAndSizeForPartition().first);
                    total_number_of_bytes += table_merge_tree->totalBytes(settings).value();
                    total_number_of_rows += table_merge_tree->totalRows(settings).value();
-                    total_number_of_parts += table_merge_tree->getPartsCount();
+                    total_number_of_parts += table_merge_tree->getActivePartsCount();
                }

                if (StorageReplicatedMergeTree * table_replicated_merge_tree = typeid_cast<StorageReplicatedMergeTree *>(table.get()))
--- a/src/Interpreters/TreeOptimizer.cpp
+++ b/src/Interpreters/TreeOptimizer.cpp
@ -23,6 +23,8 @@
 #include <Interpreters/Context.h>
 #include <Interpreters/ExternalDictionariesLoader.h>
 #include <Interpreters/GatherFunctionQuantileVisitor.h>
+#include <Interpreters/RewriteSumIfFunctionVisitor.h>
+#include <Interpreters/RewriteArrayExistsFunctionVisitor.h>

 #include <Parsers/ASTExpressionList.h>
 #include <Parsers/ASTFunction.h>
@ -37,7 +39,6 @@
 #include <Functions/UserDefined/UserDefinedExecutableFunctionFactory.h>
 #include <Storages/IStorage.h>

-#include <Interpreters/RewriteSumIfFunctionVisitor.h>

 namespace DB
 {
@ -658,6 +659,12 @@ void optimizeSumIfFunctions(ASTPtr & query)
    RewriteSumIfFunctionVisitor(data).visit(query);
 }

+void optimizeArrayExistsFunctions(ASTPtr & query)
+{
+    RewriteArrayExistsFunctionVisitor::Data data = {};
+    RewriteArrayExistsFunctionVisitor(data).visit(query);
+}
+
 void optimizeMultiIfToIf(ASTPtr & query)
 {
    OptimizeMultiIfToIfVisitor::Data data;
@ -790,6 +797,9 @@ void TreeOptimizer::apply(ASTPtr & query, TreeRewriterResult & result,
    if (settings.optimize_rewrite_sum_if_to_count_if)
        optimizeSumIfFunctions(query);

+    if (settings.optimize_rewrite_array_exists_to_has)
+        optimizeArrayExistsFunctions(query);
+
    /// Remove injective functions inside uniq
    if (settings.optimize_injective_functions_inside_uniq)
        optimizeInjectiveFunctionsInsideUniq(query, context);
@ -799,9 +809,7 @@ void TreeOptimizer::apply(ASTPtr & query, TreeRewriterResult & result,
        && !select_query->group_by_with_totals
        && !select_query->group_by_with_rollup
        && !select_query->group_by_with_cube)
-    {
        optimizeAggregateFunctionsOfGroupByKeys(select_query, query);
-    }

    /// Remove duplicate ORDER BY and DISTINCT from subqueries.
    if (settings.optimize_duplicate_order_by_and_distinct)
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@ -313,7 +313,8 @@ void addAggregationStep(QueryPlan & query_plan,
        std::move(sort_description_for_merging),
        std::move(group_by_sort_description),
        query_analysis_result.aggregation_should_produce_results_in_order_of_bucket_number,
-        settings.enable_memory_bound_merging_of_aggregation_results);
+        settings.enable_memory_bound_merging_of_aggregation_results,
+        settings.force_aggregation_in_order);
    query_plan.addStep(std::move(aggregating_step));
 }

--- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp
+++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp
@ -602,6 +602,17 @@ void ValuesBlockInputFormat::readSuffix()

 void ValuesBlockInputFormat::resetParser()
 {
+    if (got_exception)
+    {
+        /// In case of exception always reset the templates and parser type,
+        /// because they may be in the invalid state.
+        for (size_t i = 0; i < num_columns; ++i)
+        {
+            templates[i].reset();
+            parser_type_for_column[i] = ParserType::Streaming;
+        }
+    }
+
    IInputFormat::resetParser();
    // I'm not resetting parser modes here.
    // There is a good chance that all messages have the same format.
--- a/src/Processors/ISource.cpp
+++ b/src/Processors/ISource.cpp
@ -110,6 +110,7 @@ void ISource::work()
    catch (...)
    {
        finished = true;
+        got_exception = true;
        throw;
    }
 }
--- a/src/Processors/PingPongProcessor.cpp
+++ b/src/Processors/PingPongProcessor.cpp
@ -1,11 +1,12 @@
 #include <Processors/PingPongProcessor.h>

+
 namespace DB
 {

 /// Create list with `num_ports` of regular ports and 1 auxiliary port with empty header.
 template <typename T> requires std::is_same_v<T, InputPorts> || std::is_same_v<T, OutputPorts>
-static T createPortsWithSpecial(const Block & header, size_t num_ports)
+static T createPortsWithExtra(const Block & header, size_t num_ports)
 {
    T res(num_ports, header);
    res.emplace_back(Block());
@ -13,8 +14,8 @@ static T createPortsWithSpecial(const Block & header, size_t num_ports)
 }

 PingPongProcessor::PingPongProcessor(const Block & header, size_t num_ports, Order order_)
-    : IProcessor(createPortsWithSpecial<InputPorts>(header, num_ports),
-                 createPortsWithSpecial<OutputPorts>(header, num_ports))
+    : IProcessor(createPortsWithExtra<InputPorts>(header, num_ports),
+                 createPortsWithExtra<OutputPorts>(header, num_ports))
    , aux_in_port(inputs.back())
    , aux_out_port(outputs.back())
    , order(order_)
@ -195,4 +196,10 @@ std::pair<InputPort *, OutputPort *> PingPongProcessor::getAuxPorts()
    return std::make_pair(&aux_in_port, &aux_out_port);
 }

+bool ReadHeadBalancedProcessor::consume(const Chunk & chunk)
+{
+    data_consumed += chunk.getNumRows();
+    return data_consumed > size_to_wait;
+}
+
 }
--- a/src/Processors/PingPongProcessor.h
+++ b/src/Processors/PingPongProcessor.h
@ -91,11 +91,7 @@ public:

    String getName() const override { return "ReadHeadBalancedProcessor"; }

-    bool consume(const Chunk & chunk) override
-    {
-        data_consumed += chunk.getNumRows();
-        return data_consumed > size_to_wait;
-    }
+    bool consume(const Chunk & chunk) override;

 private:
    size_t data_consumed;
--- a/src/Processors/QueryPlan/AggregatingStep.cpp
+++ b/src/Processors/QueryPlan/AggregatingStep.cpp
@ -5,12 +5,14 @@
 #include <DataTypes/DataTypeFixedString.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <Functions/FunctionFactory.h>
+#include <IO/Operators.h>
 #include <Interpreters/Aggregator.h>
 #include <Interpreters/Context.h>
 #include <Processors/Merges/AggregatingSortedTransform.h>
 #include <Processors/Merges/FinishAggregatingInOrderTransform.h>
 #include <Processors/QueryPlan/AggregatingStep.h>
 #include <Processors/QueryPlan/IQueryPlanStep.h>
+#include <Processors/QueryPlan/SortingStep.h>
 #include <Processors/Transforms/AggregatingInOrderTransform.h>
 #include <Processors/Transforms/AggregatingTransform.h>
 #include <Processors/Transforms/CopyTransform.h>
@ -18,7 +20,6 @@
 #include <Processors/Transforms/MemoryBoundMerging.h>
 #include <Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
-#include <IO/Operators.h>
 #include <Common/JSONBuilder.h>

 namespace DB
@ -101,7 +102,8 @@ AggregatingStep::AggregatingStep(
    SortDescription sort_description_for_merging_,
    SortDescription group_by_sort_description_,
    bool should_produce_results_in_order_of_bucket_number_,
-    bool memory_bound_merging_of_aggregation_results_enabled_)
+    bool memory_bound_merging_of_aggregation_results_enabled_,
+    bool explicit_sorting_required_for_aggregation_in_order_)
    : ITransformingStep(
        input_stream_,
        appendGroupingColumn(params_.getHeader(input_stream_.header, final_), params_.keys, !grouping_sets_params_.empty(), group_by_use_nulls_),
@ -120,11 +122,13 @@ AggregatingStep::AggregatingStep(
    , group_by_sort_description(std::move(group_by_sort_description_))
    , should_produce_results_in_order_of_bucket_number(should_produce_results_in_order_of_bucket_number_)
    , memory_bound_merging_of_aggregation_results_enabled(memory_bound_merging_of_aggregation_results_enabled_)
+    , explicit_sorting_required_for_aggregation_in_order(explicit_sorting_required_for_aggregation_in_order_)
 {
    if (memoryBoundMergingWillBeUsed())
    {
        output_stream->sort_description = group_by_sort_description;
        output_stream->sort_scope = DataStream::SortScope::Global;
+        output_stream->has_single_port = true;
    }
 }

@ -139,6 +143,8 @@ void AggregatingStep::applyOrder(SortDescription sort_description_for_merging_,
        output_stream->sort_scope = DataStream::SortScope::Global;
        output_stream->has_single_port = true;
    }
+
+    explicit_sorting_required_for_aggregation_in_order = false;
 }

 void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings)
@ -333,6 +339,15 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B

    if (!sort_description_for_merging.empty())
    {
+        /// We don't rely here on input_stream.sort_description because it is not correctly propagated for now in all cases
+        /// see https://github.com/ClickHouse/ClickHouse/pull/45892#discussion_r1094503048
+        if (explicit_sorting_required_for_aggregation_in_order)
+        {
+            /// We don't really care about optimality of this sorting, because it's required only in fairly marginal cases.
+            SortingStep::fullSortStreams(
+                pipeline, SortingStep::Settings(params.max_block_size), sort_description_for_merging, 0 /* limit */);
+        }
+
        if (pipeline.getNumStreams() > 1)
        {
            /** The pipeline is the following:
--- a/Show More
+++ b/Show More