Merge branch 'master' into improve_decimal

.github/workflows/pull_request.yml

@@ -984,6 +984,75 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
+############################################################################################
+#################################### INSTALL PACKAGES ######################################
+############################################################################################
+  InstallPackagesTestRelease:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/test_install
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Install packages (amd64)
+          REPO_COPY=${{runner.temp}}/test_install/ClickHouse
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Test packages installation
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 install_check.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  InstallPackagesTestAarch64:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, style-checker-aarch64]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/test_install
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Install packages (arm64)
+          REPO_COPY=${{runner.temp}}/test_install/ClickHouse
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Test packages installation
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 install_check.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
 ##############################################################################################
 ########################### FUNCTIONAl STATELESS TESTS #######################################
 ##############################################################################################
@@ -2813,6 +2882,217 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
+  # Parallel replicas
+  FunctionalStatefulTestDebugParallelReplicas:
+    needs: [BuilderDebDebug]
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/stateful_debug
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Stateful tests (debug, ParallelReplicas)
+          REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
+          KILL_TIMEOUT=3600
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Functional test
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  FunctionalStatefulTestUBsanParallelReplicas:
+    needs: [BuilderDebUBsan]
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/stateful_ubsan
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Stateful tests (ubsan, ParallelReplicas)
+          REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse
+          KILL_TIMEOUT=3600
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Functional test
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  FunctionalStatefulTestMsanParallelReplicas:
+    needs: [BuilderDebMsan]
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/stateful_msan
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Stateful tests (msan, ParallelReplicas)
+          REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse
+          KILL_TIMEOUT=3600
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Functional test
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  FunctionalStatefulTestTsanParallelReplicas:
+    needs: [BuilderDebTsan]
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/stateful_tsan
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Stateful tests (tsan, ParallelReplicas)
+          REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse
+          KILL_TIMEOUT=3600
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Functional test
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  FunctionalStatefulTestAsanParallelReplicas:
+    needs: [BuilderDebAsan]
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/stateful_debug
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Stateful tests (asan, ParallelReplicas)
+          REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
+          KILL_TIMEOUT=3600
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Functional test
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  FunctionalStatefulTestReleaseParallelReplicas:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/stateful_release
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Stateful tests (release, ParallelReplicas)
+          REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse
+          KILL_TIMEOUT=3600
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Functional test
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
 ##############################################################################################
 ######################################### STRESS TESTS #######################################
 ##############################################################################################

.gitmodules

@@ -257,6 +257,9 @@
 [submodule "contrib/qpl"]
 	path = contrib/qpl
 	url = https://github.com/intel/qpl
+[submodule "contrib/idxd-config"]
+	path = contrib/idxd-config
+	url = https://github.com/intel/idxd-config
 [submodule "contrib/wyhash"]
 	path = contrib/wyhash
 	url = https://github.com/wangyi-fudan/wyhash

contrib/aws

@@ -1 +1 @@
-Subproject commit 4a12641211d4dbc8e2fdb2dd0f1eea0927db9252
+Subproject commit 06a6610e6fb3385e22ad85014a67aa307825ffb1

contrib/azure

@@ -1 +1 @@
-Subproject commit ea8c3044f43f5afa7016d2d580ed201f495d7e94
+Subproject commit e4fcdfc81e337e589ce231a452dcc280fcbb3f99

contrib/idxd-config

@@ -0,0 +1 @@
+Subproject commit f6605c41a735e3fdfef2d2d18655a33af6490b99

contrib/qpl

@@ -1 +1 @@
-Subproject commit becb7a1b15bdb4845ec3721a550707ffa51d029d
+Subproject commit d75a29d95d8a548297fce3549d21020005364dc8

contrib/qpl-cmake/CMakeLists.txt

@@ -10,11 +10,30 @@ if (NOT ENABLE_QPL)
     return()
 endif()
 
+## QPL has build dependency on libaccel-config. Here is to build libaccel-config which is required by QPL.
+## libaccel-config is the utility library for controlling and configuring Intel® In-Memory Analytics Accelerator (Intel® IAA).
+set (LIBACCEL_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/idxd-config")
+set (UUID_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl-cmake")
+set (LIBACCEL_HEADER_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl-cmake/idxd-header")
+set (SRCS
+    "${LIBACCEL_SOURCE_DIR}/accfg/lib/libaccfg.c"
+    "${LIBACCEL_SOURCE_DIR}/util/log.c"
+    "${LIBACCEL_SOURCE_DIR}/util/sysfs.c"
+)
+
+add_library(accel-config ${SRCS})
+
+target_compile_options(accel-config PRIVATE "-D_GNU_SOURCE")
+
+target_include_directories(accel-config BEFORE
+    PRIVATE ${UUID_DIR}
+    PRIVATE ${LIBACCEL_HEADER_DIR}
+    PRIVATE ${LIBACCEL_SOURCE_DIR})
+
+## QPL build start here.
 set (QPL_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl")
 set (QPL_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl/sources")
 set (QPL_BINARY_DIR "${ClickHouse_BINARY_DIR}/build/contrib/qpl")
-set (UUID_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl-cmake")
-
 set (EFFICIENT_WAIT OFF)
 set (BLOCK_ON_FAULT ON)
 set (LOG_HW_INIT OFF)
@@ -315,13 +334,8 @@ target_compile_definitions(_qpl
         PRIVATE -DQPL_BADARG_CHECK
         PUBLIC -DENABLE_QPL_COMPRESSION)
 
-find_library(LIBACCEL accel-config)
-if(NOT LIBACCEL)
-    message(FATAL_ERROR "Please install QPL dependency library:libaccel-config from https://github.com/intel/idxd-config")
-endif()
-
 target_link_libraries(_qpl
-        PRIVATE ${LIBACCEL}
+        PRIVATE accel-config
         PRIVATE ${CMAKE_DL_LIBS})
 
 add_library (ch_contrib::qpl ALIAS _qpl)

contrib/qpl-cmake/idxd-header/config.h

@@ -0,0 +1,159 @@
+/* config.h.  Generated from config.h.in by configure.  */
+/* config.h.in.  Generated from configure.ac by autoheader.  */
+
+/* Define if building universal (internal helper macro) */
+/* #undef AC_APPLE_UNIVERSAL_BUILD */
+
+/* Debug messages. */
+/* #undef ENABLE_DEBUG */
+
+/* Documentation / man pages. */
+/* #define ENABLE_DOCS */
+
+/* System logging. */
+#define ENABLE_LOGGING 1
+
+/* accfg test support */
+/* #undef ENABLE_TEST */
+
+/* Define to 1 if big-endian-arch */
+/* #undef HAVE_BIG_ENDIAN */
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#define HAVE_DLFCN_H 1
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#define HAVE_INTTYPES_H 1
+
+/* Define to 1 if you have the <linux/version.h> header file. */
+#define HAVE_LINUX_VERSION_H 1
+
+/* Define to 1 if little-endian-arch */
+#define HAVE_LITTLE_ENDIAN 1
+
+/* Define to 1 if you have the <memory.h> header file. */
+#define HAVE_MEMORY_H 1
+
+/* Define to 1 if you have the `secure_getenv' function. */
+#define HAVE_SECURE_GETENV 1
+
+/* Define to 1 if you have statement expressions. */
+#define HAVE_STATEMENT_EXPR 1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#define HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the <strings.h> header file. */
+#define HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the <string.h> header file. */
+#define HAVE_STRING_H 1
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#define HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#define HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if typeof works with your compiler. */
+#define HAVE_TYPEOF 1
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if using libuuid */
+#define HAVE_UUID 1
+
+/* Define to 1 if you have the `__secure_getenv' function. */
+/* #undef HAVE___SECURE_GETENV */
+
+/* Define to the sub-directory where libtool stores uninstalled libraries. */
+#define LT_OBJDIR ".libs/"
+
+/* Name of package */
+#define PACKAGE "accel-config"
+
+/* Define to the address where bug reports for this package should be sent. */
+#define PACKAGE_BUGREPORT "linux-dsa@lists.01.org"
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME "accel-config"
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING "accel-config 3.5.2.gitf6605c41"
+
+/* Define to the one symbol short name of this package. */
+#define PACKAGE_TARNAME "accel-config"
+
+/* Define to the home page for this package. */
+#define PACKAGE_URL "https://github.com/xxx/accel-config"
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION "3.5.2.gitf6605c41"
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
+
+/* Enable extensions on AIX 3, Interix.  */
+#ifndef _ALL_SOURCE
+# define _ALL_SOURCE 1
+#endif
+/* Enable GNU extensions on systems that have them.  */
+#ifndef _GNU_SOURCE
+# define _GNU_SOURCE 1
+#endif
+/* Enable threading extensions on Solaris.  */
+#ifndef _POSIX_PTHREAD_SEMANTICS
+# define _POSIX_PTHREAD_SEMANTICS 1
+#endif
+/* Enable extensions on HP NonStop.  */
+#ifndef _TANDEM_SOURCE
+# define _TANDEM_SOURCE 1
+#endif
+/* Enable general extensions on Solaris.  */
+#ifndef __EXTENSIONS__
+# define __EXTENSIONS__ 1
+#endif
+
+
+/* Version number of package */
+#define VERSION "3.5.2.gitf6605c41"
+
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+   significant byte first (like Motorola and SPARC, unlike Intel). */
+#if defined AC_APPLE_UNIVERSAL_BUILD
+# if defined __BIG_ENDIAN__
+#  define WORDS_BIGENDIAN 1
+# endif
+#else
+# ifndef WORDS_BIGENDIAN
+/* #  undef WORDS_BIGENDIAN */
+# endif
+#endif
+
+/* Enable large inode numbers on Mac OS X 10.5.  */
+#ifndef _DARWIN_USE_64_BIT_INODE
+# define _DARWIN_USE_64_BIT_INODE 1
+#endif
+
+/* Number of bits in a file offset, on hosts where this is settable. */
+/* #undef _FILE_OFFSET_BITS */
+
+/* Define for large files, on AIX-style hosts. */
+/* #undef _LARGE_FILES */
+
+/* Define to 1 if on MINIX. */
+/* #undef _MINIX */
+
+/* Define to 2 if the system does not provide POSIX.1 features except with
+   this defined. */
+/* #undef _POSIX_1_SOURCE */
+
+/* Define to 1 if you need to in order for `stat' and other things to work. */
+/* #undef _POSIX_SOURCE */
+
+/* Define to __typeof__ if your compiler spells it that way. */
+/* #undef typeof */

docker/images.json

@@ -134,6 +134,14 @@
         "name": "clickhouse/keeper-jepsen-test",
         "dependent": []
     },
+    "docker/test/install/deb": {
+        "name": "clickhouse/install-deb-test",
+        "dependent": []
+    },
+    "docker/test/install/rpm": {
+        "name": "clickhouse/install-rpm-test",
+        "dependent": []
+    },
     "docker/docs/builder": {
         "name": "clickhouse/docs-builder",
         "dependent": [

docker/server/Dockerfile.alpine

@@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="23.1.2.9"
+ARG VERSION="23.1.3.5"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
 # user/group precreated explicitly with fixed uid/gid on purpose.

docker/server/Dockerfile.ubuntu

@@ -21,7 +21,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
 
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="23.1.2.9"
+ARG VERSION="23.1.3.5"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
 # set non-empty deb_location_url url to create a docker image

docker/test/fasttest/run.sh

@@ -231,6 +231,7 @@ function run_tests
         --hung-check
         --fast-tests-only
         --no-random-settings
+        --no-random-merge-tree-settings
         --no-long
         --testname
         --shard

docker/test/install/deb/Dockerfile

@@ -0,0 +1,64 @@
+FROM ubuntu:22.04
+
+# The Dockerfile is nicely borrowed from
+# https://github.com/lionelnicolas/docker-ubuntu-systemd/blob/83aa3249146f5df264fe45353f79fc76eb1e42d7/Dockerfile
+
+ENV \
+  DEBIAN_FRONTEND=noninteractive \
+  LANG=C.UTF-8 \
+  container=docker \
+  init=/lib/systemd/systemd
+
+# install systemd packages
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends \
+    systemd \
+    && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists
+
+# configure systemd
+# remove systemd 'wants' triggers
+# remove everything except tmpfiles setup in sysinit target
+# remove UTMP updater service
+# disable /tmp mount
+# fix missing BPF firewall support warning
+# just for cosmetics, fix "not-found" entries while using "systemctl --all"
+RUN \
+  find \
+    /etc/systemd/system/*.wants/* \
+    /lib/systemd/system/multi-user.target.wants/* \
+    /lib/systemd/system/sockets.target.wants/*initctl* \
+    ! -type d \
+    -delete && \
+  find \
+    /lib/systemd/system/sysinit.target.wants \
+    ! -type d \
+    ! -name '*systemd-tmpfiles-setup*' \
+    -delete && \
+  find \
+    /lib/systemd \
+    -name systemd-update-utmp-runlevel.service \
+    -delete && \
+  rm -vf /usr/share/systemd/tmp.mount && \
+  sed -ri '/^IPAddressDeny/d' /lib/systemd/system/systemd-journald.service && \
+  for MATCH in \
+    plymouth-start.service \
+    plymouth-quit-wait.service \
+    syslog.socket \
+    syslog.service \
+    display-manager.service \
+    systemd-sysusers.service \
+    tmp.mount \
+    systemd-udevd.service \
+    ; do \
+      grep -rn --binary-files=without-match  ${MATCH} /lib/systemd/ | cut -d: -f1 | xargs sed -ri 's/(.*=.*)'${MATCH}'(.*)/\1\2/'; \
+  done && \
+  systemctl disable ondemand.service && \
+  systemctl set-default multi-user.target
+
+VOLUME ["/run", "/run/lock"]
+
+STOPSIGNAL SIGRTMIN+3
+
+ENTRYPOINT ["/lib/systemd/systemd"]

docker/test/install/rpm/Dockerfile

@@ -0,0 +1,55 @@
+FROM centos:8
+
+# The Dockerfile is nicely borrowed from
+# https://github.com/lionelnicolas/docker-ubuntu-systemd/blob/83aa3249146f5df264fe45353f79fc76eb1e42d7/Dockerfile
+
+ENV \
+  LANG=C.UTF-8 \
+  container=docker \
+  init=/lib/systemd/systemd
+
+# configure systemd
+# remove systemd 'wants' triggers
+# remove everything except tmpfiles setup in sysinit target
+# remove UTMP updater service
+# disable /tmp mount
+# fix missing BPF firewall support warning
+# just for cosmetics, fix "not-found" entries while using "systemctl --all"
+RUN \
+  find \
+    /etc/systemd/system/*.wants/ \
+    /lib/systemd/system/multi-user.target.wants/ \
+    /lib/systemd/system/local-fs.target.wants/ \
+    /lib/systemd/system/sockets.target.wants/*initctl* \
+    ! -type d \
+    -delete && \
+  find \
+    /lib/systemd/system/sysinit.target.wants \
+    ! -type d \
+    ! -name '*systemd-tmpfiles-setup*' \
+    -delete && \
+  find \
+    /lib/systemd \
+    -name systemd-update-utmp-runlevel.service \
+    -delete && \
+  rm -vf /usr/share/systemd/tmp.mount && \
+  sed -ri '/^IPAddressDeny/d' /lib/systemd/system/systemd-journald.service && \
+  for MATCH in \
+    plymouth-start.service \
+    plymouth-quit-wait.service \
+    syslog.socket \
+    syslog.service \
+    display-manager.service \
+    systemd-sysusers.service \
+    tmp.mount \
+    systemd-udevd.service \
+    ; do \
+      grep -rn --binary-files=without-match  ${MATCH} /lib/systemd/ | cut -d: -f1 | xargs sed -ri 's/(.*=.*)'${MATCH}'(.*)/\1\2/'; \
+  done && \
+  systemctl set-default multi-user.target
+
+VOLUME ["/run", "/run/lock"]
+
+STOPSIGNAL SIGRTMIN+3
+
+ENTRYPOINT ["/lib/systemd/systemd"]

docker/test/stateful/run.sh

@@ -126,13 +126,16 @@ function run_tests()
     fi
 
     set +e
-    clickhouse-test -j 2 --testname --shard --zookeeper --check-zookeeper-session --no-stateless --hung-check --print-time \
+
-        --skip 00168_parallel_processing_on_replicas "${ADDITIONAL_OPTIONS[@]}" \
+    if [[ -n "$USE_PARALLEL_REPLICAS" ]] && [[ "$USE_PARALLEL_REPLICAS" -eq 1 ]]; then
+        clickhouse-test --client="clickhouse-client --use_hedged_requests=0  --allow_experimental_parallel_reading_from_replicas=1 \
+            --max_parallel_replicas=100 --cluster_for_parallel_replicas='parallel_replicas'" \
+            -j 2 --testname --shard --zookeeper --check-zookeeper-session --no-stateless --no-parallel-replicas --hung-check --print-time "${ADDITIONAL_OPTIONS[@]}" \
         "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
-
+    else
-    clickhouse-test --timeout 1200 --testname --shard --zookeeper --check-zookeeper-session --no-stateless --hung-check --print-time \
+        clickhouse-test -j 2 --testname --shard --zookeeper --check-zookeeper-session --no-stateless --hung-check --print-time "${ADDITIONAL_OPTIONS[@]}" \
-    00168_parallel_processing_on_replicas "${ADDITIONAL_OPTIONS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a test_output/test_result.txt
+        "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
-
+    fi
     set -e
 }
 

docker/test/stateless/run.sh

@@ -134,9 +134,9 @@ function run_tests()
 
     set +e
     clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
-            --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
+        --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
-        | ts '%Y-%m-%d %H:%M:%S' \
+    | ts '%Y-%m-%d %H:%M:%S' \
-        | tee -a test_output/test_result.txt
+    | tee -a test_output/test_result.txt
     set -e
 }
 

docker/test/stress/run.sh

@@ -13,15 +13,28 @@ sysctl kernel.core_pattern='core.%e.%p-%P'
 
 OK="\tOK\t\\N\t"
 FAIL="\tFAIL\t\\N\t"
+
+FAILURE_CONTEXT_LINES=50
+FAILURE_CONTEXT_MAX_LINE_WIDTH=400
+
 function escaped()
 {
     # That's the simplest way I found to escape a string in bash. Yep, bash is the most convenient programming language.
-    clickhouse local -S 's String' --input-format=LineAsString -q "select * from table format CustomSeparated settings format_custom_row_after_delimiter='\\\\\\\\n'"
+    # Also limit lines width just in case (too long lines are not really useful usually)
+    clickhouse local -S 's String' --input-format=LineAsString -q "select substr(s, 1, $FAILURE_CONTEXT_MAX_LINE_WIDTH)
+      from table format CustomSeparated settings format_custom_row_after_delimiter='\\\\\\\\n'"
 }
-
 function head_escaped()
 {
-    head -50 $1 | escaped
+    head -n $FAILURE_CONTEXT_LINES $1 | escaped
+}
+function unts()
+{
+    grep -Po "[0-9][0-9]:[0-9][0-9] \K.*"
+}
+function trim_server_logs()
+{
+    head -n $FAILURE_CONTEXT_LINES "/test_output/$1" | grep -Eo " \[ [0-9]+ \] \{.*" | escaped
 }
 
 function install_packages()
@@ -167,7 +180,7 @@ function start()
         then
             echo "Cannot start clickhouse-server"
             rg --text "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt ||:
-            echo -e "Cannot start clickhouse-server$FAIL$(head_escaped /test_output/application_errors.txt)" >> /test_output/test_results.tsv
+            echo -e "Cannot start clickhouse-server$FAIL$(trim_server_logs application_errors.txt)" >> /test_output/test_results.tsv
             cat /var/log/clickhouse-server/stdout.log
             tail -n100 /var/log/clickhouse-server/stderr.log
             tail -n100000 /var/log/clickhouse-server/clickhouse-server.log | rg -F -v -e '<Warning> RaftInstance:' -e '<Information> RaftInstance' | tail -n100
@@ -389,7 +402,7 @@ start
 
 # NOTE Hung check is implemented in docker/tests/stress/stress
 rg -Fa "No queries hung" /test_output/test_results.tsv | grep -Fa "OK" \
-  || echo -e "Hung check failed, possible deadlock found (see hung_check.log)$FAIL$(head_escaped /test_output/hung_check.log)"
+  || echo -e "Hung check failed, possible deadlock found (see hung_check.log)$FAIL$(head_escaped /test_output/hung_check.log | unts)"
 
 stop
 mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.stress.log
@@ -402,7 +415,7 @@ start
 
 clickhouse-client --query "SELECT 'Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \
     || (rg --text "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt \
-    && echo -e "Server failed to start (see application_errors.txt and clickhouse-server.clean.log)$FAIL$(head_escaped /test_output/application_errors.txt)" \
+    && echo -e "Server failed to start (see application_errors.txt and clickhouse-server.clean.log)$FAIL$(trim_server_logs application_errors.txt)" \
     >> /test_output/test_results.tsv)
 
 stop
@@ -435,7 +448,7 @@ rg -Fa "Code: 49. DB::Exception: " /var/log/clickhouse-server/clickhouse-server*
 
 # No such key errors
 rg --text "Code: 499.*The specified key does not exist" /var/log/clickhouse-server/clickhouse-server*.log > /test_output/no_such_key_errors.txt \
-    && echo -e "S3_ERROR No such key thrown (see clickhouse-server.log or no_such_key_errors.txt)$FAIL$(head_escaped /test_output/no_such_key_errors.txt)" >> /test_output/test_results.tsv \
+    && echo -e "S3_ERROR No such key thrown (see clickhouse-server.log or no_such_key_errors.txt)$FAIL$(trim_server_logs no_such_key_errors.txt)" >> /test_output/test_results.tsv \
     || echo -e "No lost s3 keys$OK" >> /test_output/test_results.tsv
 
 # Remove file no_such_key_errors.txt if it's empty
@@ -448,7 +461,7 @@ rg -Fa "########################################" /var/log/clickhouse-server/cli
 
 # It also checks for crash without stacktrace (printed by watchdog)
 rg -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server*.log > /test_output/fatal_messages.txt \
-    && echo -e "Fatal message in clickhouse-server.log (see fatal_messages.txt)$FAIL$(head_escaped /test_output/fatal_messages.txt)" >> /test_output/test_results.tsv \
+    && echo -e "Fatal message in clickhouse-server.log (see fatal_messages.txt)$FAIL$(trim_server_logs fatal_messages.txt)" >> /test_output/test_results.tsv \
     || echo -e "No fatal messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv
 
 # Remove file fatal_messages.txt if it's empty
@@ -457,8 +470,13 @@ rg -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server*.log > /test_out
 rg -Fa "########################################" /test_output/* > /dev/null \
     && echo -e "Killed by signal (output files)$FAIL" >> /test_output/test_results.tsv
 
+function get_gdb_log_context()
+{
+    rg -A50 -Fa " received signal " /test_output/gdb.log | head_escaped
+}
+
 rg -Fa " received signal " /test_output/gdb.log > /dev/null \
-    && echo -e "Found signal in gdb.log$FAIL$(rg -A50 -Fa " received signal " /test_output/gdb.log | escaped)" >> /test_output/test_results.tsv
+    && echo -e "Found signal in gdb.log$FAIL$(get_gdb_log_context)" >> /test_output/test_results.tsv
 
 if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
     echo -e "Backward compatibility check\n"
@@ -579,7 +597,7 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
         start 500
         clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \
             || (rg --text "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log >> /test_output/bc_check_application_errors.txt \
-            && echo -e "Backward compatibility check: Server failed to start$FAIL$(head_escaped /test_output/bc_check_application_errors.txt)" >> /test_output/test_results.tsv)
+            && echo -e "Backward compatibility check: Server failed to start$FAIL$(trim_server_logs bc_check_application_errors.txt)" >> /test_output/test_results.tsv)
 
         clickhouse-client --query="SELECT 'Server version: ', version()"
 
@@ -634,7 +652,7 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
                    -e "Session expired" \
                    -e "TOO_MANY_PARTS" \
             /var/log/clickhouse-server/clickhouse-server.backward.dirty.log | rg -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
-            && echo -e "Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)$FAIL$(head_escaped /test_output/bc_check_error_messages.txt)" \
+            && echo -e "Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)$FAIL$(trim_server_logs bc_check_error_messages.txt)" \
                 >> /test_output/test_results.tsv \
             || echo -e "Backward compatibility check: No Error messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv
 
@@ -657,7 +675,7 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
         # Logical errors
         echo "Check for Logical errors in server log:"
         rg -Fa -A20 "Code: 49. DB::Exception:" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_logical_errors.txt \
-            && echo -e "Backward compatibility check: Logical error thrown (see clickhouse-server.log or bc_check_logical_errors.txt)$FAIL$(head_escaped /test_output/bc_check_logical_errors.txt)" \
+            && echo -e "Backward compatibility check: Logical error thrown (see clickhouse-server.log or bc_check_logical_errors.txt)$FAIL$(trim_server_logs bc_check_logical_errors.txt)" \
                 >> /test_output/test_results.tsv \
             || echo -e "Backward compatibility check: No logical errors$OK" >> /test_output/test_results.tsv
 
@@ -672,7 +690,7 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
         # It also checks for crash without stacktrace (printed by watchdog)
         echo "Check for Fatal message in server log:"
         rg -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_fatal_messages.txt \
-            && echo -e "Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)$FAIL$(head_escaped /test_output/bc_check_fatal_messages.txt)" \
+            && echo -e "Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)$FAIL$(trim_server_logs bc_check_fatal_messages.txt)" \
                 >> /test_output/test_results.tsv \
             || echo -e "Backward compatibility check: No fatal messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv
 

docker/test/util/process_functional_tests_result.py

@@ -85,8 +85,16 @@ def process_test_log(log_path):
             if DATABASE_SIGN in line:
                 test_end = True
 
+    # Python does not support TSV, so we have to escape '\t' and '\n' manually
+    # and hope that complex escape sequences will not break anything
     test_results = [
-        (test[0], test[1], test[2], "".join(test[3])[:4096]) for test in test_results
+        (
+            test[0],
+            test[1],
+            test[2],
+            "".join(test[3])[:4096].replace("\t", "\\t").replace("\n", "\\n"),
+        )
+        for test in test_results
     ]
 
     return (

docs/changelogs/v23.1.3.5-stable.md

@@ -0,0 +1,17 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.1.3.5-stable (548b494bcce) FIXME as compared to v23.1.2.9-stable (8dfb1700858)
+
+#### Bug Fix (user-visible misbehavior in official stable or prestable release)
+
+* Backported in [#45896](https://github.com/ClickHouse/ClickHouse/issues/45896): Bugfix IPv6 parser for mixed ip4 address with missed first octet (like `::.1.2.3`). [#45871](https://github.com/ClickHouse/ClickHouse/pull/45871) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Get rid of progress timestamps in release publishing [#45818](https://github.com/ClickHouse/ClickHouse/pull/45818) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+

docs/en/development/tests.md

@@ -16,6 +16,11 @@ Tests are located in `queries` directory. There are two subdirectories: `statele
 
 Each test can be one of two types: `.sql` and `.sh`. `.sql` test is the simple SQL script that is piped to `clickhouse-client --multiquery`. `.sh` test is a script that is run by itself. SQL tests are generally preferable to `.sh` tests. You should use `.sh` tests only when you have to test some feature that cannot be exercised from pure SQL, such as piping some input data into `clickhouse-client` or testing `clickhouse-local`.
 
+:::note
+A common mistake when testing data types `DateTime` and `DateTime64` is assuming that the server uses a specific time zone (e.g. "UTC"). This is not the case, time zones in CI test runs
+are deliberately randomized. The easiest workaround is to specify the time zone for test values explicitly, e.g. `toDateTime64(val, 3, 'Europe/Amsterdam')`.
+:::
+
 ### Running a Test Locally {#functional-test-locally}
 
 Start the ClickHouse server locally, listening on the default port (9000). To

docs/en/engines/table-engines/integrations/rabbitmq.md

@@ -77,9 +77,12 @@ Optional parameters:
 - `rabbitmq_password` - RabbitMQ password.
 - `rabbitmq_commit_on_select` - Commit messages when select query is made. Default: `false`.
 - `rabbitmq_max_rows_per_message` — The maximum number of rows written in one RabbitMQ message for row-based formats. Default : `1`.
+- `rabbitmq_empty_queue_backoff_start` — A start backoff point to reschedule read if the rabbitmq queue is empty.
+- `rabbitmq_empty_queue_backoff_end` — An end backoff point to reschedule read if the rabbitmq queue is empty.
 
 
-SSL connection:
+
+  * [ ] SSL connection:
 
 Use either `rabbitmq_secure = 1` or `amqps` in connection address: `rabbitmq_address = 'amqps://guest:guest@localhost/vhost'`.
 The default behaviour of the used library is not to check if the created TLS connection is sufficiently secure. Whether the certificate is expired, self-signed, missing or invalid: the connection is simply permitted. More strict checking of certificates can possibly be implemented in the future.

docs/en/engines/table-engines/mergetree-family/invertedindexes.md

@@ -86,9 +86,9 @@ CREATE TABLE hackernews (
     author String,
     timestamp DateTime,
     comment String,
-	dead UInt8,
+    dead UInt8,
-	parent UInt64,
+    parent UInt64,
-	poll UInt64,
+    poll UInt64,
     children Array(UInt32),
     url String,
     score UInt32,
@@ -171,9 +171,23 @@ WHERE hasToken(lower(comment), 'clickhouse')
 1 row in set. Elapsed: 0.747 sec. Processed 4.49 million rows, 1.77 GB (6.01 million rows/s., 2.37 GB/s.)
 ```
 
+We can also search for one or all of multiple terms, i.e., disjunctions or conjunctions:
+
+```sql
+-- multiple OR'ed terms
+SELECT count(*)
+FROM hackernews
+WHERE multiSearchAny(lower(comment), ['oltp', 'olap']);
+
+-- multiple AND'ed terms
+SELECT count(*)
+FROM hackernews
+WHERE hasToken(lower(comment), 'avx') AND hasToken(lower(comment), 'sve');
+```
+
 :::note
 Unlike other secondary indices, inverted indexes (for now) map to row numbers (row ids) instead of granule ids. The reason for this design
 is performance. In practice, users often search for multiple terms at once. For example, filter predicate `WHERE s LIKE '%little%' OR s LIKE
 '%big%'` can be evaluated directly using an inverted index by forming the union of the row id lists for terms "little" and "big". This also
 means that the parameter `GRANULARITY` supplied to index creation has no meaning (it may be removed from the syntax in the future).
-:::
+:::

docs/en/operations/server-configuration-parameters/settings.md

@@ -1279,7 +1279,9 @@ The following settings are available:
 -   `size`: The maximum cache size in bytes. 0 means the query cache is disabled. Default value: `1073741824` (1 GiB).
 -   `max_entries`: The maximum number of `SELECT` query results stored in the cache. Default value: `1024`.
 -   `max_entry_size`: The maximum size in bytes `SELECT` query results may have to be saved in the cache. Default value: `1048576` (1 MiB).
--   `max_entry_records`: The maximum number of records `SELECT` query results may have to be saved in the cache. Default value: `30000000` (30 mil).
+-   `max_entry_rows`: The maximum number of rows `SELECT` query results may have to be saved in the cache. Default value: `30000000` (30 mil).
+
+Changed settings take effect immediately.
 
 :::warning
 Data for the query cache is allocated in DRAM. If memory is scarce, make sure to set a small value for `size` or disable the query cache altogether.
@@ -1292,7 +1294,7 @@ Data for the query cache is allocated in DRAM. If memory is scarce, make sure to
     <size>1073741824</size>
     <max_entries>1024</max_entries>
     <max_entry_size>1048576</max_entry_size>
-    <max_entry_records>30000000</max_entry_records>
+    <max_entry_rows>30000000</max_entry_rows>
 </query_cache>
 ```
 

docs/en/sql-reference/functions/comparison-functions.md

@@ -21,14 +21,14 @@ For example, you can’t compare a date with a string. You have to use a functio
 
 Strings are compared by bytes. A shorter string is smaller than all strings that start with it and that contain at least one more character.
 
-## equals, a = b and a == b operator
+### equals, a `=` b and a `==` b operator
 
-## notEquals, a != b and a \<\> b operator
+### notEquals, a `!=` b and a `<>` b operator
 
-## less, \< operator
+### less, `<` operator
 
-## greater, \> operator
+### greater, `>` operator
 
-## lessOrEquals, \<= operator
+### lessOrEquals, `<=` operator
 
-## greaterOrEquals, \>= operator
+### greaterOrEquals, `>=` operator

docs/en/sql-reference/statements/delete.md

@@ -2,15 +2,16 @@
 slug: /en/sql-reference/statements/delete
 sidebar_position: 36
 sidebar_label: DELETE
+description: Lightweight deletes simplify the process of deleting data from the database.
+keywords: [delete]
+title: DELETE Statement
 ---
 
-# DELETE Statement
-
 ``` sql
 DELETE FROM [db.]table [ON CLUSTER cluster] [WHERE expr]
 ```
 
-`DELETE FROM` removes rows from table `[db.]table` that match expression `expr`. The deleted rows are marked as deleted immediately and will be automatically filtered out of all subsequent queries. Cleanup of data happens asynchronously in background. This feature is only available for MergeTree table engine family.
+`DELETE FROM` removes rows from the table `[db.]table` that match the expression `expr`. The deleted rows are marked as deleted immediately and will be automatically filtered out of all subsequent queries. Cleanup of data happens asynchronously in the background. This feature is only available for the MergeTree table engine family.
 
 For example, the following query deletes all rows from the `hits` table where the `Title` column contains the text `hello`:
 
@@ -32,7 +33,7 @@ SET allow_experimental_lightweight_delete = true;
 An [alternative way to delete rows](./alter/delete.md) in ClickHouse is `ALTER TABLE ... DELETE`, which might be more efficient if you do bulk deletes only occasionally and don't need the operation to be applied instantly. In most use cases the new lightweight `DELETE FROM` behavior will be considerably faster.
 
 :::warning
-Even though deletes are becoming more lightweight in ClickHouse, they should still not be used as aggressively as on an OLTP system. Ligthweight deletes are currently efficient for wide parts, but for compact parts they can be a heavyweight operation, and it may be better to use `ALTER TABLE` for some scenarios.
+Even though deletes are becoming more lightweight in ClickHouse, they should still not be used as aggressively as on an OLTP system. Lightweight deletes are currently efficient for wide parts, but for compact parts, they can be a heavyweight operation, and it may be better to use `ALTER TABLE` for some scenarios.
 :::
 
 :::note
@@ -41,3 +42,34 @@ Even though deletes are becoming more lightweight in ClickHouse, they should sti
 grant ALTER DELETE ON db.table to username;
 ```
 :::
+
+## Lightweight Delete Internals
+
+The idea behind Lightweight Delete is that when a `DELETE FROM table ...` query is executed ClickHouse only saves a mask where each row is marked as either “existing” or as “deleted”. Those “deleted” rows become invisible for subsequent queries, but physically the rows are removed only later by subsequent merges. Writing this mask is usually much more lightweight than what is done by `ALTER table DELETE ...` query.
+
+### How it is implemented
+The mask is implemented as a hidden `_row_exists` system column that stores True for all visible rows and False for deleted ones. This column is only present in a part if some rows in this part were deleted. In other words, the column is not persisted when it has all values equal to True.
+
+## SELECT query
+When the column is present `SELECT ... FROM table WHERE condition` query internally is extended by an additional predicate on `_row_exists` and becomes similar to 
+```sql
+    SELECT ... FROM table PREWHERE _row_exists WHERE condition
+```
+At execution time the column `_row_exists` is read to figure out which rows are not visible and if there are many deleted rows it can figure out which granules can be fully skipped when reading the rest of the columns.
+
+## DELETE query
+`DELETE FROM table WHERE condition` is translated into `ALTER table UPDATE _row_exists = 0 WHERE condition` mutation. Internally this mutation is executed in 2 steps:
+1. `SELECT count() FROM table WHERE condition` for each individual part to figure out if the part is affected.
+2. Mutate affected parts, and make hardlinks for unaffected parts. Mutating a part in fact only writes `_row_exists` column and just hardlinks all other columns’ files in the case of Wide parts. But for Compact parts, all columns are rewritten because they all are stored together in one file.
+
+So if we compare Lightweight Delete to `ALTER DELETE` in the first step they both do the same thing to figure out which parts are affected, but in the second step `ALTER DELETE` does much more work because it reads and rewrites all columns’ files for the affected parts.
+
+With the described implementation now we can see what can negatively affect 'DELETE FROM' execution time:
+- Heavy WHERE condition in DELETE query
+- Mutations queue filled with other mutations, because all mutations on a table are executed sequentially
+- Table having a very large number of data parts
+- Having a lot of data in Compact parts—in a Compact part, all columns are stored in one file.
+
+:::note
+This implementation might change in the future.
+:::

packages/build

@@ -102,7 +102,8 @@ done
 EOF
     chmod +x "$PKG_PATH/install/doinst.sh"
     if [ -f "$PKG_PATH/DEBIAN/postinst" ]; then
-        tail +2 "$PKG_PATH/DEBIAN/postinst" >> "$PKG_PATH/install/doinst.sh"
+        # we don't need debconf source in doinst in any case
+        tail +2 "$PKG_PATH/DEBIAN/postinst" | grep -v debconf/confmodule >> "$PKG_PATH/install/doinst.sh"
     fi
     rm -rf "$PKG_PATH/DEBIAN"
     if [ -f "/usr/bin/pigz" ]; then

packages/clickhouse-keeper.postinstall

@@ -0,0 +1,46 @@
+#!/bin/sh
+set -e
+# set -x
+
+PROGRAM=clickhouse-keeper
+KEEPER_USER=${KEEPER_USER:=clickhouse}
+KEEPER_GROUP=${KEEPER_GROUP:=clickhouse}
+# Please note that we don't support paths with whitespaces. This is rather ignorant.
+KEEPER_CONFDIR=${KEEPER_CONFDIR:=/etc/$PROGRAM}
+KEEPER_DATADIR=${KEEPER_DATADIR:=/var/lib/clickhouse}
+KEEPER_LOGDIR=${KEEPER_LOGDIR:=/var/log/$PROGRAM}
+
+[ -f /usr/share/debconf/confmodule ] && . /usr/share/debconf/confmodule
+[ -f /etc/default/clickhouse-keeper ] && . /etc/default/clickhouse-keeper
+
+if [ ! -f "/etc/debian_version" ]; then
+    not_deb_os=1
+fi
+
+if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then
+    if ! getent group "${KEEPER_GROUP}" > /dev/null 2>&1 ; then
+        groupadd --system "${KEEPER_GROUP}"
+    fi
+    GID=$(getent group "${KEEPER_GROUP}" | cut -d: -f 3)
+    if ! id "${KEEPER_USER}" > /dev/null 2>&1 ; then
+        adduser --system --home /dev/null --no-create-home \
+            --gid "${GID}" --shell /bin/false \
+            "${KEEPER_USER}"
+    fi
+
+    chown -R "${KEEPER_USER}:${KEEPER_GROUP}" "${KEEPER_CONFDIR}"
+    chmod 0755 "${KEEPER_CONFDIR}"
+
+    if ! [ -d "${KEEPER_DATADIR}" ]; then
+        mkdir -p "${KEEPER_DATADIR}"
+        chown -R "${KEEPER_USER}:${KEEPER_GROUP}" "${KEEPER_DATADIR}"
+        chmod 0700 "${KEEPER_DATADIR}"
+    fi
+
+    if ! [ -d "${KEEPER_LOGDIR}" ]; then
+        mkdir -p "${KEEPER_LOGDIR}"
+        chown -R "${KEEPER_USER}:${KEEPER_GROUP}" "${KEEPER_LOGDIR}"
+        chmod 0770 "${KEEPER_LOGDIR}"
+    fi
+fi
+# vim: ts=4: sw=4: sts=4: expandtab

packages/clickhouse-keeper.service

@@ -0,0 +1,27 @@
+[Unit]
+Description=ClickHouse Keeper - zookeeper compatible distributed coordination server
+Requires=network-online.target
+# NOTE: that After/Wants=time-sync.target is not enough, you need to ensure
+# that the time was adjusted already, if you use systemd-timesyncd you are
+# safe, but if you use ntp or some other daemon, you should configure it
+# additionaly.
+After=time-sync.target network-online.target
+Wants=time-sync.target
+
+[Service]
+Type=simple
+User=clickhouse
+Group=clickhouse
+Restart=always
+RestartSec=30
+RuntimeDirectory=%p  # %p is resolved to the systemd unit name
+ExecStart=/usr/bin/clickhouse-keeper --config=/etc/clickhouse-keeper/keeper_config.xml --pid-file=%t/%p/%p.pid
+# Minus means that this file is optional.
+EnvironmentFile=-/etc/default/%p
+LimitCORE=infinity
+LimitNOFILE=500000
+CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE CAP_NET_BIND_SERVICE
+
+[Install]
+# ClickHouse should not start from the rescue shell (rescue.target).
+WantedBy=multi-user.target

packages/clickhouse-keeper.yaml

@@ -30,6 +30,8 @@ contents:
   type: config|noreplace
 - src: root/usr/bin/clickhouse-keeper
   dst: /usr/bin/clickhouse-keeper
+- src: clickhouse-keeper.service
+  dst: /lib/systemd/system/clickhouse-keeper.service
 # docs
 - src: ../AUTHORS
   dst: /usr/share/doc/clickhouse-keeper/AUTHORS
@@ -39,3 +41,6 @@ contents:
   dst: /usr/share/doc/clickhouse-keeper/LICENSE
 - src: ../README.md
   dst: /usr/share/doc/clickhouse-keeper/README.md
+
+scripts:
+  postinstall: ./clickhouse-keeper.postinstall

packages/clickhouse-server.postinstall

@@ -11,8 +11,6 @@ CLICKHOUSE_DATADIR=${CLICKHOUSE_DATADIR:=/var/lib/clickhouse}
 CLICKHOUSE_LOGDIR=${CLICKHOUSE_LOGDIR:=/var/log/clickhouse-server}
 CLICKHOUSE_BINDIR=${CLICKHOUSE_BINDIR:=/usr/bin}
 CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM:=clickhouse}
-EXTRACT_FROM_CONFIG=${CLICKHOUSE_GENERIC_PROGRAM}-extract-from-config
-CLICKHOUSE_CONFIG=$CLICKHOUSE_CONFDIR/config.xml
 CLICKHOUSE_PIDDIR=/var/run/$PROGRAM
 
 [ -f /usr/share/debconf/confmodule ] && . /usr/share/debconf/confmodule

packages/clickhouse-server.service

@@ -17,10 +17,10 @@ User=clickhouse
 Group=clickhouse
 Restart=always
 RestartSec=30
-RuntimeDirectory=clickhouse-server
+RuntimeDirectory=%p  # %p is resolved to the systemd unit name
-ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml --pid-file=/run/clickhouse-server/clickhouse-server.pid
+ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml --pid-file=%t/%p/%p.pid
 # Minus means that this file is optional.
-EnvironmentFile=-/etc/default/clickhouse
+EnvironmentFile=-/etc/default/%p
 LimitCORE=infinity
 LimitNOFILE=500000
 CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE CAP_NET_BIND_SERVICE

programs/benchmark/Benchmark.cpp

@@ -474,7 +474,7 @@ private:
         executor.sendQuery(ClientInfo::QueryKind::INITIAL_QUERY);
 
         ProfileInfo info;
-        while (Block block = executor.read())
+        while (Block block = executor.readBlock())
             info.update(block);
 
         executor.finish();

programs/client/Client.cpp

@@ -1,3 +1,4 @@
+#include <boost/algorithm/string/join.hpp>
 #include <cstdlib>
 #include <fcntl.h>
 #include <map>
@@ -538,24 +539,28 @@ void Client::connect()
 // Prints changed settings to stderr. Useful for debugging fuzzing failures.
 void Client::printChangedSettings() const
 {
-    const auto & changes = global_context->getSettingsRef().changes();
+    auto print_changes = [](const auto & changes, std::string_view settings_name)
-    if (!changes.empty())
     {
-        fmt::print(stderr, "Changed settings: ");
+        if (!changes.empty())
-        for (size_t i = 0; i < changes.size(); ++i)
         {
-            if (i)
+            fmt::print(stderr, "Changed {}: ", settings_name);
+            for (size_t i = 0; i < changes.size(); ++i)
             {
-                fmt::print(stderr, ", ");
+                if (i)
+                    fmt::print(stderr, ", ");
+                fmt::print(stderr, "{} = '{}'", changes[i].name, toString(changes[i].value));
             }
-            fmt::print(stderr, "{} = '{}'", changes[i].name, toString(changes[i].value));
+
+            fmt::print(stderr, "\n");
         }
-        fmt::print(stderr, "\n");
+        else
-    }
+        {
-    else
+            fmt::print(stderr, "No changed {}.\n", settings_name);
-    {
+        }
-        fmt::print(stderr, "No changed settings.\n");
+    };
-    }
+
+    print_changes(global_context->getSettingsRef().changes(), "settings");
+    print_changes(cmd_merge_tree_settings.changes(), "MergeTree settings");
 }
 
 
@@ -1352,6 +1357,8 @@ void Client::readArguments(
             }
             else if (arg == "--allow_repeated_settings")
                 allow_repeated_settings = true;
+            else if (arg == "--allow_merge_tree_settings")
+                allow_merge_tree_settings = true;
             else
                 common_arguments.emplace_back(arg);
         }

programs/copier/ClusterCopier.cpp

@@ -2040,7 +2040,7 @@ UInt64 ClusterCopier::executeQueryOnCluster(
 
                 while (true)
                 {
-                    auto block = remote_query_executor->read();
+                    auto block = remote_query_executor->readBlock();
                     if (!block)
                         break;
                 }

programs/keeper/Keeper.cpp

@@ -362,6 +362,7 @@ try
     else
         path = std::filesystem::path{KEEPER_DEFAULT_PATH};
 
+    std::filesystem::create_directories(path);
 
     /// Check that the process user id matches the owner of the data.
     const auto effective_user_id = geteuid();

programs/local/CMakeLists.txt

@@ -19,6 +19,9 @@ target_link_libraries(clickhouse-local-lib PRIVATE clickhouse-server-lib)
 if (TARGET ch_rust::skim)
     target_link_libraries(clickhouse-local-lib PRIVATE ch_rust::skim)
 endif()
+if (TARGET ch_contrib::azure_sdk)
+    target_link_libraries(clickhouse-local-lib PRIVATE ch_contrib::azure_sdk)
+endif()
 
 # Always use internal readpassphrase
 target_link_libraries(clickhouse-local-lib PRIVATE readpassphrase)

programs/local/LocalServer.cpp

@@ -51,6 +51,10 @@
     #include <Functions/getFuzzerData.h>
 #endif
 
+#if USE_AZURE_BLOB_STORAGE
+#   include <azure/storage/common/internal/xml_wrapper.hpp>
+#endif
+
 namespace fs = std::filesystem;
 
 
@@ -115,6 +119,14 @@ void LocalServer::initialize(Poco::Util::Application & self)
         config().getUInt("thread_pool_queue_size", 10000)
     );
 
+#if USE_AZURE_BLOB_STORAGE
+    /// See the explanation near the same line in Server.cpp
+    GlobalThreadPool::instance().addOnDestroyCallback([]
+    {
+        Azure::Storage::_internal::XmlGlobalDeinitialize();
+    });
+#endif
+
     IOThreadPool::initialize(
         config().getUInt("max_io_thread_pool_size", 100),
         config().getUInt("max_io_thread_pool_free_size", 0),

programs/server/CMakeLists.txt

@@ -27,6 +27,9 @@ set (CLICKHOUSE_SERVER_LINK
 if (TARGET ch_contrib::jemalloc)
     list(APPEND CLICKHOUSE_SERVER_LINK PRIVATE ch_contrib::jemalloc)
 endif()
+if (TARGET ch_contrib::azure_sdk)
+    list(APPEND CLICKHOUSE_SERVER_LINK PRIVATE ch_contrib::azure_sdk)
+endif()
 
 clickhouse_program_add(server)
 

programs/server/Server.cpp

@@ -82,9 +82,7 @@
 #include <Common/ThreadFuzzer.h>
 #include <Common/getHashOfLoadedBinary.h>
 #include <Common/filesystemHelpers.h>
-#if USE_BORINGSSL
 #include <Compression/CompressionCodecEncrypted.h>
-#endif
 #include <Server/HTTP/HTTPServerConnectionFactory.h>
 #include <Server/MySQLHandlerFactory.h>
 #include <Server/PostgreSQLHandlerFactory.h>
@@ -128,6 +126,10 @@
 #    include <jemalloc/jemalloc.h>
 #endif
 
+#if USE_AZURE_BLOB_STORAGE
+#   include <azure/storage/common/internal/xml_wrapper.hpp>
+#endif
+
 namespace CurrentMetrics
 {
     extern const Metric Revision;
@@ -750,6 +752,19 @@ try
         config().getUInt("max_thread_pool_free_size", 1000),
         config().getUInt("thread_pool_queue_size", 10000));
 
+#if USE_AZURE_BLOB_STORAGE
+    /// It makes sense to deinitialize libxml after joining of all threads
+    /// in global pool because libxml uses thread-local memory allocations via
+    /// 'pthread_key_create' and 'pthread_setspecific' which should be deallocated
+    /// at 'pthread_exit'. Deinitialization of libxml leads to call of 'pthread_key_delete'
+    /// and if it is done before joining of threads, allocated memory will not be freed
+    /// and there may be memory leaks in threads that used libxml.
+    GlobalThreadPool::instance().addOnDestroyCallback([]
+    {
+        Azure::Storage::_internal::XmlGlobalDeinitialize();
+    });
+#endif
+
     IOThreadPool::initialize(
         config().getUInt("max_io_thread_pool_size", 100),
         config().getUInt("max_io_thread_pool_free_size", 0),
@@ -1331,9 +1346,8 @@ try
 
             global_context->updateStorageConfiguration(*config);
             global_context->updateInterserverCredentials(*config);
-#if USE_BORINGSSL
+            global_context->updateQueryCacheConfiguration(*config);
             CompressionCodecEncrypted::Configuration::instance().tryLoad(*config, "encryption_codecs");
-#endif
 #if USE_SSL
             CertificateReloader::instance().tryLoad(*config);
 #endif
@@ -1517,13 +1531,7 @@ try
         global_context->setMMappedFileCache(mmap_cache_size);
 
     /// A cache for query results.
-    size_t query_cache_size = config().getUInt64("query_cache.size", 1_GiB);
+    global_context->setQueryCache(config());
-    if (query_cache_size)
-        global_context->setQueryCache(
-            query_cache_size,
-            config().getUInt64("query_cache.max_entries", 1024),
-            config().getUInt64("query_cache.max_entry_size", 1_MiB),
-            config().getUInt64("query_cache.max_entry_records", 30'000'000));
 
 #if USE_EMBEDDED_COMPILER
     /// 128 MB
@@ -1547,10 +1555,8 @@ try
         global_context->getMergeTreeSettings().sanityCheck(background_pool_tasks);
         global_context->getReplicatedMergeTreeSettings().sanityCheck(background_pool_tasks);
     }
-#if USE_BORINGSSL
     /// try set up encryption. There are some errors in config, error will be printed and server wouldn't start.
     CompressionCodecEncrypted::Configuration::instance().load(config(), "encryption_codecs");
-#endif
 
     SCOPE_EXIT({
         async_metrics.stop();

programs/server/config.xml

@@ -854,6 +854,51 @@
                 </replica>
             </shard-->
         </test_cluster_one_shard_three_replicas_localhost>
+	<parallel_replicas>
+		<shard>
+            <internal_replication>false</internal_replication>
+            <replica>
+                <host>127.0.0.1</host>
+                <port>9000</port>
+            </replica>
+            <replica>
+                <host>127.0.0.2</host>
+                <port>9000</port>
+            </replica>
+            <replica>
+                <host>127.0.0.3</host>
+                <port>9000</port>
+            </replica>
+            <replica>
+                <host>127.0.0.4</host>
+                <port>9000</port>
+            </replica>
+            <replica>
+                <host>127.0.0.5</host>
+                <port>9000</port>
+            </replica>
+            <replica>
+                <host>127.0.0.6</host>
+                <port>9000</port>
+            </replica>
+            <replica>
+                <host>127.0.0.7</host>
+                <port>9000</port>
+            </replica>
+            <replica>
+                <host>127.0.0.8</host>
+                <port>9000</port>
+            </replica>
+            <replica>
+                <host>127.0.0.9</host>
+                <port>9000</port>
+            </replica>
+            <replica>
+                <host>127.0.0.10</host>
+                <port>9000</port>
+            </replica>
+        </shard>
+	</parallel_replicas>
         <test_cluster_two_shards_localhost>
              <shard>
                  <replica>
@@ -1471,7 +1516,7 @@
     <!--     <size>1073741824</size> -->
     <!--     <max_entries>1024</max_entries> -->
     <!--     <max_entry_size>1048576</max_entry_size> -->
-    <!--     <max_entry_records>30000000</max_entry_records> -->
+    <!--     <max_entry_rows>30000000</max_entry_rows> -->
     <!-- </query_cache> -->
 
     <!-- Uncomment if enable merge tree metadata cache -->

rust/skim/src/lib.rs

@@ -11,11 +11,26 @@ mod ffi {
 
 struct Item {
     text: String,
+    orig_text: String,
+}
+impl Item {
+    fn new(text: String) -> Self {
+        return Self{
+            // Text that will be shown should not contains new lines since in this case skim may
+            // live some symbols on the screen, and this looks odd.
+            text: text.replace("\n", " "),
+            orig_text: text,
+        };
+    }
 }
 impl SkimItem for Item {
     fn text(&self) -> Cow<str> {
         return Cow::Borrowed(&self.text);
     }
+
+    fn output(&self) -> Cow<str> {
+        return Cow::Borrowed(&self.orig_text);
+    }
 }
 
 fn skim(prefix: &CxxString, words: &CxxVector<CxxString>) -> Result<String, String> {
@@ -34,7 +49,7 @@ fn skim(prefix: &CxxString, words: &CxxVector<CxxString>) -> Result<String, Stri
 
     let (tx, rx): (SkimItemSender, SkimItemReceiver) = unbounded();
     for word in words {
-        tx.send(Arc::new(Item{ text: word.to_string() })).unwrap();
+        tx.send(Arc::new(Item::new(word.to_string()))).unwrap();
     }
     // so that skim could know when to stop waiting for more items.
     drop(tx);

src/Access/Common/AccessType.h

@@ -146,6 +146,7 @@ enum class AccessType
     M(SYSTEM_DROP_COMPILED_EXPRESSION_CACHE, "SYSTEM DROP COMPILED EXPRESSION, DROP COMPILED EXPRESSION CACHE, DROP COMPILED EXPRESSIONS", GLOBAL, SYSTEM_DROP_CACHE) \
     M(SYSTEM_DROP_FILESYSTEM_CACHE, "SYSTEM DROP FILESYSTEM CACHE, DROP FILESYSTEM CACHE", GLOBAL, SYSTEM_DROP_CACHE) \
     M(SYSTEM_DROP_SCHEMA_CACHE, "SYSTEM DROP SCHEMA CACHE, DROP SCHEMA CACHE", GLOBAL, SYSTEM_DROP_CACHE) \
+    M(SYSTEM_DROP_S3_CLIENT_CACHE, "SYSTEM DROP S3 CLIENT, DROP S3 CLIENT CACHE", GLOBAL, SYSTEM_DROP_CACHE) \
     M(SYSTEM_DROP_CACHE, "DROP CACHE", GROUP, SYSTEM) \
     M(SYSTEM_RELOAD_CONFIG, "RELOAD CONFIG", GLOBAL, SYSTEM_RELOAD) \
     M(SYSTEM_RELOAD_USERS, "RELOAD USERS", GLOBAL, SYSTEM_RELOAD) \

src/AggregateFunctions/AggregateFunctionMLMethod.cpp

@@ -247,15 +247,8 @@ void Adam::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac)
     if (adam_rhs.average_gradient.empty())
         return;
 
-    if (average_gradient.empty())
+    average_gradient.resize(adam_rhs.average_gradient.size(), Float64{0.0});
-    {
+    average_squared_gradient.resize(adam_rhs.average_squared_gradient.size(), Float64{0.0});
-        if (!average_squared_gradient.empty() ||
-                adam_rhs.average_gradient.size() != adam_rhs.average_squared_gradient.size())
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Average_gradient and average_squared_gradient must have same size");
-
-        average_gradient.resize(adam_rhs.average_gradient.size(), Float64{0.0});
-        average_squared_gradient.resize(adam_rhs.average_squared_gradient.size(), Float64{0.0});
-    }
 
     for (size_t i = 0; i < average_gradient.size(); ++i)
     {
@@ -268,14 +261,8 @@ void Adam::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac)
 
 void Adam::update(UInt64 batch_size, std::vector<Float64> & weights, Float64 & bias, Float64 learning_rate, const std::vector<Float64> & batch_gradient)
 {
-    if (average_gradient.empty())
+    average_gradient.resize(batch_gradient.size(), Float64{0.0});
-    {
+    average_squared_gradient.resize(batch_gradient.size(), Float64{0.0});
-        if (!average_squared_gradient.empty())
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Average_gradient and average_squared_gradient must have same size");
-
-        average_gradient.resize(batch_gradient.size(), Float64{0.0});
-        average_squared_gradient.resize(batch_gradient.size(), Float64{0.0});
-    }
 
     for (size_t i = 0; i != average_gradient.size(); ++i)
     {
@@ -328,8 +315,7 @@ void Nesterov::write(WriteBuffer & buf) const
 void Nesterov::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac)
 {
     const auto & nesterov_rhs = static_cast<const Nesterov &>(rhs);
-    if (accumulated_gradient.empty())
+    accumulated_gradient.resize(nesterov_rhs.accumulated_gradient.size(), Float64{0.0});
-        accumulated_gradient.resize(nesterov_rhs.accumulated_gradient.size(), Float64{0.0});
 
     for (size_t i = 0; i < accumulated_gradient.size(); ++i)
     {
@@ -339,10 +325,7 @@ void Nesterov::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac
 
 void Nesterov::update(UInt64 batch_size, std::vector<Float64> & weights, Float64 & bias, Float64 learning_rate, const std::vector<Float64> & batch_gradient)
 {
-    if (accumulated_gradient.empty())
+    accumulated_gradient.resize(batch_gradient.size(), Float64{0.0});
-    {
-        accumulated_gradient.resize(batch_gradient.size(), Float64{0.0});
-    }
 
     for (size_t i = 0; i < batch_gradient.size(); ++i)
     {
@@ -402,10 +385,7 @@ void Momentum::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac
 void Momentum::update(UInt64 batch_size, std::vector<Float64> & weights, Float64 & bias, Float64 learning_rate, const std::vector<Float64> & batch_gradient)
 {
     /// batch_size is already checked to be greater than 0
-    if (accumulated_gradient.empty())
+    accumulated_gradient.resize(batch_gradient.size(), Float64{0.0});
-    {
-        accumulated_gradient.resize(batch_gradient.size(), Float64{0.0});
-    }
 
     for (size_t i = 0; i < batch_gradient.size(); ++i)
     {

src/AggregateFunctions/AggregateFunctionMLMethod.h

@@ -149,9 +149,11 @@ public:
 class Momentum : public IWeightsUpdater
 {
 public:
-    Momentum() = default;
 
-    explicit Momentum(Float64 alpha_) : alpha(alpha_) {}
+    explicit Momentum(size_t num_params, Float64 alpha_ = 0.1) : alpha(alpha_)
+    {
+        accumulated_gradient.resize(num_params + 1, 0);
+    }
 
     void update(UInt64 batch_size, std::vector<Float64> & weights, Float64 & bias, Float64 learning_rate, const std::vector<Float64> & batch_gradient) override;
 
@@ -170,9 +172,10 @@ private:
 class Nesterov : public IWeightsUpdater
 {
 public:
-    Nesterov() = default;
+    explicit Nesterov(size_t num_params, Float64 alpha_ = 0.9) : alpha(alpha_)
-
+    {
-    explicit Nesterov(Float64 alpha_) : alpha(alpha_) {}
+        accumulated_gradient.resize(num_params + 1, 0);
+    }
 
     void addToBatch(
         std::vector<Float64> & batch_gradient,
@@ -201,10 +204,14 @@ private:
 class Adam : public IWeightsUpdater
 {
 public:
-    Adam()
+    Adam(size_t num_params)
     {
         beta1_powered = beta1;
         beta2_powered = beta2;
+
+
+        average_gradient.resize(num_params + 1, 0);
+        average_squared_gradient.resize(num_params + 1, 0);
     }
 
     void addToBatch(
@@ -338,11 +345,11 @@ public:
         if (weights_updater_name == "SGD")
             new_weights_updater = std::make_shared<StochasticGradientDescent>();
         else if (weights_updater_name == "Momentum")
-            new_weights_updater = std::make_shared<Momentum>();
+            new_weights_updater = std::make_shared<Momentum>(param_num);
         else if (weights_updater_name == "Nesterov")
-            new_weights_updater = std::make_shared<Nesterov>();
+            new_weights_updater = std::make_shared<Nesterov>(param_num);
         else if (weights_updater_name == "Adam")
-            new_weights_updater = std::make_shared<Adam>();
+            new_weights_updater = std::make_shared<Adam>(param_num);
         else
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Illegal name of weights updater (should have been checked earlier)");
 

src/Backups/BackupIO_S3.cpp

@@ -9,13 +9,12 @@
 #include <IO/WriteBufferFromS3.h>
 #include <IO/HTTPHeaderEntries.h>
 #include <IO/S3/copyS3File.h>
+#include <IO/S3/Client.h>
+
 #include <Poco/Util/AbstractConfiguration.h>
 
 #include <aws/core/auth/AWSCredentials.h>
-#include <aws/s3/S3Client.h>
+
-#include <aws/s3/model/DeleteObjectRequest.h>
-#include <aws/s3/model/DeleteObjectsRequest.h>
-#include <aws/s3/model/ListObjectsRequest.h>
 #include <filesystem>
 
 
@@ -31,7 +30,7 @@ namespace ErrorCodes
 
 namespace
 {
-    std::shared_ptr<Aws::S3::S3Client>
+    std::shared_ptr<S3::Client>
     makeS3Client(const S3::URI & s3_uri, const String & access_key_id, const String & secret_access_key, const ContextPtr & context)
     {
         auto settings = context->getStorageS3Settings().getSettings(s3_uri.uri.toString());
@@ -71,9 +70,9 @@ namespace
                 context->getConfigRef().getBool("s3.use_insecure_imds_request", false)));
     }
 
-    Aws::Vector<Aws::S3::Model::Object> listObjects(Aws::S3::S3Client & client, const S3::URI & s3_uri, const String & file_name)
+    Aws::Vector<Aws::S3::Model::Object> listObjects(S3::Client & client, const S3::URI & s3_uri, const String & file_name)
     {
-        Aws::S3::Model::ListObjectsRequest request;
+        S3::ListObjectsRequest request;
         request.SetBucket(s3_uri.bucket);
         request.SetPrefix(fs::path{s3_uri.key} / file_name);
         request.SetMaxKeys(1);
@@ -228,7 +227,7 @@ std::unique_ptr<WriteBuffer> BackupWriterS3::writeFile(const String & file_name)
 
 void BackupWriterS3::removeFile(const String & file_name)
 {
-    Aws::S3::Model::DeleteObjectRequest request;
+    S3::DeleteObjectRequest request;
     request.SetBucket(s3_uri.bucket);
     request.SetKey(fs::path(s3_uri.key) / file_name);
     auto outcome = client->DeleteObject(request);
@@ -285,7 +284,7 @@ void BackupWriterS3::removeFilesBatch(const Strings & file_names)
 
         Aws::S3::Model::Delete delkeys;
         delkeys.SetObjects(current_chunk);
-        Aws::S3::Model::DeleteObjectsRequest request;
+        S3::DeleteObjectsRequest request;
         request.SetBucket(s3_uri.bucket);
         request.SetDelete(delkeys);
 

src/Backups/BackupIO_S3.h

@@ -7,7 +7,6 @@
 #include <IO/ReadSettings.h>
 #include <IO/S3Common.h>
 #include <Storages/StorageS3Settings.h>
-#include <aws/s3/S3Client.h>
 
 
 namespace DB
@@ -27,7 +26,7 @@ public:
 
 private:
     S3::URI s3_uri;
-    std::shared_ptr<Aws::S3::S3Client> client;
+    std::shared_ptr<S3::Client> client;
     ReadSettings read_settings;
     S3Settings::RequestSettings request_settings;
 };
@@ -73,7 +72,7 @@ private:
     void removeFilesBatch(const Strings & file_names);
 
     S3::URI s3_uri;
-    std::shared_ptr<Aws::S3::S3Client> client;
+    std::shared_ptr<S3::Client> client;
     ReadSettings read_settings;
     S3Settings::RequestSettings request_settings;
     Poco::Logger * log;

src/Backups/BackupImpl.cpp

@@ -271,16 +271,22 @@ size_t BackupImpl::getNumFiles() const
     return num_files;
 }
 
-size_t BackupImpl::getNumProcessedFiles() const
+UInt64 BackupImpl::getTotalSize() const
 {
     std::lock_guard lock{mutex};
-    return num_processed_files;
+    return total_size;
 }
 
-UInt64 BackupImpl::getProcessedFilesSize() const
+size_t BackupImpl::getNumEntries() const
 {
     std::lock_guard lock{mutex};
-    return processed_files_size;
+    return num_entries;
+}
+
+UInt64 BackupImpl::getSizeOfEntries() const
+{
+    std::lock_guard lock{mutex};
+    return size_of_entries;
 }
 
 UInt64 BackupImpl::getUncompressedSize() const
@@ -295,6 +301,18 @@ UInt64 BackupImpl::getCompressedSize() const
     return compressed_size;
 }
 
+size_t BackupImpl::getNumReadFiles() const
+{
+    std::lock_guard lock{mutex};
+    return num_read_files;
+}
+
+UInt64 BackupImpl::getNumReadBytes() const
+{
+    std::lock_guard lock{mutex};
+    return num_read_bytes;
+}
+
 void BackupImpl::writeBackupMetadata()
 {
     assert(!is_internal_backup);
@@ -323,12 +341,18 @@ void BackupImpl::writeBackupMetadata()
         }
     }
 
-    size_t index = 0;
+    num_files = all_file_infos.size();
-    for (const auto & info : all_file_infos)
+    total_size = 0;
+    num_entries = 0;
+    size_of_entries = 0;
+
+    for (size_t i = 0; i != all_file_infos.size(); ++i)
     {
-        String prefix = index ? "contents.file[" + std::to_string(index) + "]." : "contents.file.";
+        const auto & info = all_file_infos[i];
+        String prefix = i ? "contents.file[" + std::to_string(i) + "]." : "contents.file.";
         config->setString(prefix + "name", info.file_name);
         config->setUInt64(prefix + "size", info.size);
+
         if (info.size)
         {
             config->setString(prefix + "checksum", hexChecksum(info.checksum));
@@ -348,8 +372,14 @@ void BackupImpl::writeBackupMetadata()
             if (info.pos_in_archive != static_cast<size_t>(-1))
                 config->setUInt64(prefix + "pos_in_archive", info.pos_in_archive);
         }
-        increaseUncompressedSize(info);
+
-        ++index;
+        total_size += info.size;
+        bool has_entry = !deduplicate_files || (info.size && (info.size != info.base_size) && (info.data_file_name.empty() || (info.data_file_name == info.file_name)));
+        if (has_entry)
+        {
+            ++num_entries;
+            size_of_entries += info.size - info.base_size;
+        }
     }
 
     std::ostringstream stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
@@ -366,8 +396,7 @@ void BackupImpl::writeBackupMetadata()
     out->write(str.data(), str.size());
     out->finalize();
 
-    increaseUncompressedSize(str.size());
+    uncompressed_size = size_of_entries + str.size();
-    increaseProcessedSize(str.size());
 }
 
 
@@ -392,8 +421,6 @@ void BackupImpl::readBackupMetadata()
 
     String str;
     readStringUntilEOF(str, *in);
-    increaseUncompressedSize(str.size());
-    increaseProcessedSize(str.size());
     Poco::XML::DOMParser dom_parser;
     Poco::AutoPtr<Poco::XML::Document> config = dom_parser.parseMemory(str.data(), str.size());
     const Poco::XML::Node * config_root = getRootNode(config);
@@ -412,6 +439,11 @@ void BackupImpl::readBackupMetadata()
     if (config_root->getNodeByPath("base_backup_uuid"))
         base_backup_uuid = parse<UUID>(getString(config_root, "base_backup_uuid"));
 
+    num_files = 0;
+    total_size = 0;
+    num_entries = 0;
+    size_of_entries = 0;
+
     const auto * contents = config_root->getNodeByPath("contents");
     for (const Poco::XML::Node * child = contents->firstChild(); child; child = child->nextSibling())
     {
@@ -456,10 +488,20 @@ void BackupImpl::readBackupMetadata()
             }
 
             coordination->addFileInfo(info);
-            increaseUncompressedSize(info);
+
+            ++num_files;
+            total_size += info.size;
+            bool has_entry = !deduplicate_files || (info.size && (info.size != info.base_size) && (info.data_file_name.empty() || (info.data_file_name == info.file_name)));
+            if (has_entry)
+            {
+                ++num_entries;
+                size_of_entries += info.size - info.base_size;
+            }
         }
     }
 
+    uncompressed_size = size_of_entries + str.size();
+    compressed_size = uncompressed_size;
     if (!use_archives)
         setCompressedSize();
 }
@@ -612,7 +654,8 @@ BackupEntryPtr BackupImpl::readFile(const SizeAndChecksum & size_and_checksum) c
     if (open_mode != OpenMode::READ)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Backup is not opened for reading");
 
-    increaseProcessedSize(size_and_checksum.first);
+    ++num_read_files;
+    num_read_bytes += size_and_checksum.first;
 
     if (!size_and_checksum.first)
     {
@@ -780,7 +823,8 @@ void BackupImpl::writeFile(const String & file_name, BackupEntryPtr entry)
 
     {
         std::lock_guard lock{mutex};
-        increaseProcessedSize(info);
+        ++num_files;
+        total_size += info.size;
     }
 
     /// Empty file, nothing to backup
@@ -909,7 +953,7 @@ void BackupImpl::writeFile(const String & file_name, BackupEntryPtr entry)
     {
         LOG_TRACE(log, "Will copy file {}", adjusted_path);
 
-        if (!num_files_written)
+        if (!num_entries)
             checkLockFile(true);
 
         if (use_archives)
@@ -951,7 +995,12 @@ void BackupImpl::writeFile(const String & file_name, BackupEntryPtr entry)
         }
     }
 
-    ++num_files_written;
+    {
+        std::lock_guard lock{mutex};
+        ++num_entries;
+        size_of_entries += info.size - info.base_size;
+        uncompressed_size += info.size - info.base_size;
+    }
 }
 
 
@@ -981,29 +1030,6 @@ void BackupImpl::finalizeWriting()
 }
 
 
-void BackupImpl::increaseUncompressedSize(UInt64 file_size)
-{
-    uncompressed_size += file_size;
-    ++num_files;
-}
-
-void BackupImpl::increaseUncompressedSize(const FileInfo & info)
-{
-    if ((info.size > info.base_size) && (info.data_file_name.empty() || (info.data_file_name == info.file_name)))
-        increaseUncompressedSize(info.size - info.base_size);
-}
-
-void BackupImpl::increaseProcessedSize(UInt64 file_size) const
-{
-    processed_files_size += file_size;
-    ++num_processed_files;
-}
-
-void BackupImpl::increaseProcessedSize(const FileInfo & info)
-{
-    increaseProcessedSize(info.size);
-}
-
 void BackupImpl::setCompressedSize()
 {
     if (use_archives)

src/Backups/BackupImpl.h

@@ -59,10 +59,13 @@ public:
     time_t getTimestamp() const override { return timestamp; }
     UUID getUUID() const override { return *uuid; }
     size_t getNumFiles() const override;
-    size_t getNumProcessedFiles() const override;
+    UInt64 getTotalSize() const override;
-    UInt64 getProcessedFilesSize() const override;
+    size_t getNumEntries() const override;
+    UInt64 getSizeOfEntries() const override;
     UInt64 getUncompressedSize() const override;
     UInt64 getCompressedSize() const override;
+    size_t getNumReadFiles() const override;
+    UInt64 getNumReadBytes() const override;
     Strings listFiles(const String & directory, bool recursive) const override;
     bool hasFiles(const String & directory) const override;
     bool fileExists(const String & file_name) const override;
@@ -103,16 +106,6 @@ private:
     std::shared_ptr<IArchiveReader> getArchiveReader(const String & suffix) const;
     std::shared_ptr<IArchiveWriter> getArchiveWriter(const String & suffix);
 
-    /// Increases `uncompressed_size` by a specific value,
-    /// also increases `num_files` by 1.
-    void increaseUncompressedSize(UInt64 file_size);
-    void increaseUncompressedSize(const FileInfo & info);
-
-    /// Increases `num_processed_files` by a specific value,
-    /// also increases `num_processed_files` by 1.
-    void increaseProcessedSize(UInt64 file_size) const;
-    void increaseProcessedSize(const FileInfo & info);
-
     /// Calculates and sets `compressed_size`.
     void setCompressedSize();
 
@@ -129,10 +122,13 @@ private:
     std::optional<UUID> uuid;
     time_t timestamp = 0;
     size_t num_files = 0;
-    mutable size_t num_processed_files = 0;
+    UInt64 total_size = 0;
-    mutable UInt64 processed_files_size = 0;
+    size_t num_entries = 0;
+    UInt64 size_of_entries = 0;
     UInt64 uncompressed_size = 0;
     UInt64 compressed_size = 0;
+    mutable size_t num_read_files = 0;
+    mutable UInt64 num_read_bytes = 0;
     int version;
     std::optional<BackupInfo> base_backup_info;
     std::shared_ptr<const IBackup> base_backup;
@@ -141,7 +137,6 @@ private:
     std::pair<String, std::shared_ptr<IArchiveWriter>> archive_writers[2];
     String current_archive_suffix;
     String lock_file_name;
-    std::atomic<size_t> num_files_written = 0;
     bool writing_finalized = false;
     bool deduplicate_files = true;
     const Poco::Logger * log;

src/Backups/BackupsWorker.cpp

@@ -338,20 +338,20 @@ void BackupsWorker::doBackup(
         }
 
         size_t num_files = 0;
-        size_t num_processed_files = 0;
+        UInt64 total_size = 0;
+        size_t num_entries = 0;
         UInt64 uncompressed_size = 0;
         UInt64 compressed_size = 0;
-        UInt64 processed_files_size = 0;
 
         /// Finalize backup (write its metadata).
         if (!backup_settings.internal)
         {
             backup->finalizeWriting();
             num_files = backup->getNumFiles();
-            num_processed_files = backup->getNumProcessedFiles();
+            total_size = backup->getTotalSize();
+            num_entries = backup->getNumEntries();
             uncompressed_size = backup->getUncompressedSize();
             compressed_size = backup->getCompressedSize();
-            processed_files_size = backup->getProcessedFilesSize();
         }
 
         /// Close the backup.
@@ -359,7 +359,7 @@ void BackupsWorker::doBackup(
 
         LOG_INFO(log, "{} {} was created successfully", (backup_settings.internal ? "Internal backup" : "Backup"), backup_name_for_logging);
         setStatus(backup_id, BackupStatus::BACKUP_CREATED);
-        setNumFilesAndSize(backup_id, num_files, num_processed_files, processed_files_size, uncompressed_size, compressed_size);
+        setNumFilesAndSize(backup_id, num_files, total_size, num_entries, uncompressed_size, compressed_size, 0, 0);
     }
     catch (...)
     {
@@ -583,10 +583,12 @@ void BackupsWorker::doRestore(
         setNumFilesAndSize(
             restore_id,
             backup->getNumFiles(),
-            backup->getNumProcessedFiles(),
+            backup->getTotalSize(),
-            backup->getProcessedFilesSize(),
+            backup->getNumEntries(),
             backup->getUncompressedSize(),
-            backup->getCompressedSize());
+            backup->getCompressedSize(),
+            backup->getNumReadFiles(),
+            backup->getNumReadBytes());
     }
     catch (...)
     {
@@ -667,7 +669,9 @@ void BackupsWorker::setStatus(const String & id, BackupStatus status, bool throw
 }
 
 
-void BackupsWorker::setNumFilesAndSize(const String & id, size_t num_files, size_t num_processed_files, UInt64 processed_files_size, UInt64 uncompressed_size, UInt64 compressed_size)
+void BackupsWorker::setNumFilesAndSize(const OperationID & id, size_t num_files, UInt64 total_size, size_t num_entries,
+                                       UInt64 uncompressed_size, UInt64 compressed_size, size_t num_read_files, UInt64 num_read_bytes)
+
 {
     std::lock_guard lock{infos_mutex};
     auto it = infos.find(id);
@@ -676,10 +680,12 @@ void BackupsWorker::setNumFilesAndSize(const String & id, size_t num_files, size
 
     auto & info = it->second;
     info.num_files = num_files;
-    info.num_processed_files = num_processed_files;
+    info.total_size = total_size;
-    info.processed_files_size = processed_files_size;
+    info.num_entries = num_entries;
     info.uncompressed_size = uncompressed_size;
     info.compressed_size = compressed_size;
+    info.num_read_files = num_read_files;
+    info.num_read_bytes = num_read_bytes;
 }
 
 

src/Backups/BackupsWorker.h

@@ -53,23 +53,27 @@ public:
         /// Status of backup or restore operation.
         BackupStatus status;
 
-        /// Number of files in the backup (including backup's metadata; only unique files are counted).
+        /// The number of files stored in the backup.
         size_t num_files = 0;
 
-        /// Number of processed files during backup or restore process
+        /// The total size of files stored in the backup.
-        /// For restore it includes files from base backups
+        UInt64 total_size = 0;
-        size_t num_processed_files = 0;
 
-        /// Size of processed files during backup or restore
+        /// The number of entries in the backup, i.e. the number of files inside the folder if the backup is stored as a folder.
-        /// For restore in includes sizes from base backups
+        size_t num_entries = 0;
-        UInt64 processed_files_size = 0;
 
-        /// Size of all files in the backup (including backup's metadata; only unique files are counted).
+        /// The uncompressed size of the backup.
         UInt64 uncompressed_size = 0;
 
-        /// Size of the backup if it's stored as an archive; or the same as `uncompressed_size` if the backup is stored as a folder.
+        /// The compressed size of the backup.
         UInt64 compressed_size = 0;
 
+        /// Returns the number of files read during RESTORE from this backup.
+        size_t num_read_files = 0;
+
+        // Returns the total size of files read during RESTORE from this backup.
+        UInt64 num_read_bytes = 0;
+
         /// Set only if there was an error.
         std::exception_ptr exception;
         String error_message;
@@ -110,7 +114,9 @@ private:
     void addInfo(const OperationID & id, const String & name, bool internal, BackupStatus status);
     void setStatus(const OperationID & id, BackupStatus status, bool throw_if_error = true);
     void setStatusSafe(const String & id, BackupStatus status) { setStatus(id, status, false); }
-    void setNumFilesAndSize(const OperationID & id, size_t num_files, size_t num_processed_files, UInt64 processed_files_size, UInt64 uncompressed_size, UInt64 compressed_size);
+    void setNumFilesAndSize(const OperationID & id, size_t num_files, UInt64 total_size, size_t num_entries,
+                            UInt64 uncompressed_size, UInt64 compressed_size, size_t num_read_files, UInt64 num_read_bytes);
+
     std::vector<Info> getAllActiveBackupInfos() const;
     std::vector<Info> getAllActiveRestoreInfos() const;
     bool hasConcurrentBackups(const BackupSettings & backup_settings) const;

src/Backups/IBackup.h

@@ -37,21 +37,38 @@ public:
     /// Returns UUID of the backup.
     virtual UUID getUUID() const = 0;
 
-    /// Returns the number of unique files in the backup.
+    /// Returns the number of files stored in the backup. Compare with getNumEntries().
     virtual size_t getNumFiles() const = 0;
 
-    /// Returns the number of files were processed for backup or restore
+    /// Returns the total size of files stored in the backup. Compare with getTotalSizeOfEntries().
-    virtual size_t getNumProcessedFiles() const = 0;
+    virtual UInt64 getTotalSize() const  = 0;
 
-    // Returns the total size of processed files for backup or restore
+    /// Returns the number of entries in the backup, i.e. the number of files inside the folder if the backup is stored as a folder or
-    virtual UInt64 getProcessedFilesSize() const = 0;
+    /// the number of files inside the archive if the backup is stored as an archive.
+    /// It's not the same as getNumFiles() if it's an incremental backups or if it contains empty files or duplicates.
+    /// The following is always true: `getNumEntries() <= getNumFiles()`.
+    virtual size_t getNumEntries() const = 0;
 
-    /// Returns the total size of unique files in the backup.
+    /// Returns the size of entries in the backup, i.e. the total size of files inside the folder if the backup is stored as a folder or
+    /// the total size of files inside the archive if the backup is stored as an archive.
+    /// It's not the same as getTotalSize() because it doesn't include the size of duplicates and the size of files from the base backup.
+    /// The following is always true: `getSizeOfEntries() <= getTotalSize()`.
+    virtual UInt64 getSizeOfEntries() const = 0;
+
+    /// Returns the uncompressed size of the backup. It equals to `getSizeOfEntries() + size_of_backup_metadata (.backup)`
     virtual UInt64 getUncompressedSize() const = 0;
 
-    /// Returns the compressed size of the backup. If the backup is not stored as an archive it returns the same as getUncompressedSize().
+    /// Returns the compressed size of the backup. If the backup is not stored as an archive it's the same as getUncompressedSize().
     virtual UInt64 getCompressedSize() const = 0;
 
+    /// Returns the number of files read during RESTORE from this backup.
+    /// The following is always true: `getNumFilesRead() <= getNumFiles()`.
+    virtual size_t getNumReadFiles() const  = 0;
+
+    // Returns the total size of files read during RESTORE from this backup.
+    /// The following is always true: `getNumReadBytes() <= getTotalSize()`.
+    virtual UInt64 getNumReadBytes() const = 0;
+
     /// Returns names of entries stored in a specified directory in the backup.
     /// If `directory` is empty or '/' the functions returns entries in the backup's root.
     virtual Strings listFiles(const String & directory, bool recursive = false) const = 0;

src/BridgeHelper/CatBoostLibraryBridgeHelper.h

@@ -11,7 +11,7 @@
 namespace DB
 {
 
-class CatBoostLibraryBridgeHelper : public LibraryBridgeHelper
+class CatBoostLibraryBridgeHelper final : public LibraryBridgeHelper
 {
 public:
     static constexpr inline auto PING_HANDLER = "/catboost_ping";

src/BridgeHelper/ExternalDictionaryLibraryBridgeHelper.h

@@ -14,7 +14,7 @@ namespace DB
 class Pipe;
 
 // Class to access the external dictionary part of the clickhouse-library-bridge.
-class ExternalDictionaryLibraryBridgeHelper : public LibraryBridgeHelper
+class ExternalDictionaryLibraryBridgeHelper final : public LibraryBridgeHelper
 {
 
 public:

src/CMakeLists.txt

@@ -226,13 +226,7 @@ add_object_library(clickhouse_access Access)
 add_object_library(clickhouse_backups Backups)
 add_object_library(clickhouse_core Core)
 add_object_library(clickhouse_core_mysql Core/MySQL)
-if (ENABLE_OPENSSL OR ENABLE_OPENSSL_DYNAMIC)
+add_object_library(clickhouse_compression Compression)
-    add_headers_and_sources(dbms Compression)
-    list(REMOVE_ITEM dbms_headers Compression/CompressionCodecEncrypted.h)
-    list(REMOVE_ITEM dbms_sources Compression/CompressionCodecEncrypted.cpp)
-else ()
-    add_object_library(clickhouse_compression Compression)
-endif ()
 add_object_library(clickhouse_querypipeline QueryPipeline)
 add_object_library(clickhouse_datatypes DataTypes)
 add_object_library(clickhouse_datatypes_serializations DataTypes/Serializations)
@@ -343,8 +337,8 @@ set_source_files_properties(
         PROPERTIES COMPILE_FLAGS "-mwaitpkg")
 endif ()
 
-target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::re2_st)
+target_link_libraries(common PUBLIC ch_contrib::re2_st)
-target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::re2)
+target_link_libraries(common PUBLIC ch_contrib::re2)
 
 target_link_libraries(clickhouse_common_io
         PUBLIC

src/Client/ClientBase.cpp

@@ -41,6 +41,7 @@
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTColumnDeclaration.h>
+#include <Parsers/ASTFunction.h>
 #include <Parsers/Kusto/ParserKQLStatement.h>
 
 #include <Processors/Formats/Impl/NullFormat.h>
@@ -816,17 +817,15 @@ void ClientBase::processTextAsSingleQuery(const String & full_query)
 
 void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr parsed_query)
 {
-    if (fake_drop)
+    if (fake_drop && parsed_query->as<ASTDropQuery>())
-    {
+        return;
-        if (parsed_query->as<ASTDropQuery>())
+
-            return;
+    auto query = query_to_execute;
-    }
 
     /// Rewrite query only when we have query parameters.
     /// Note that if query is rewritten, comments in query are lost.
     /// But the user often wants to see comments in server logs, query log, processlist, etc.
     /// For recent versions of the server query parameters will be transferred by network and applied on the server side.
-    auto query = query_to_execute;
     if (!query_parameters.empty()
         && connection->getServerRevision(connection_parameters.timeouts) < DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS)
     {
@@ -838,6 +837,22 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa
         query = serializeAST(*parsed_query);
     }
 
+    if (allow_merge_tree_settings && parsed_query->as<ASTCreateQuery>())
+    {
+        /// Rewrite query if new settings were added.
+        if (addMergeTreeSettings(*parsed_query->as<ASTCreateQuery>()))
+        {
+            /// Replace query parameters because AST cannot be serialized otherwise.
+            if (!query_parameters.empty())
+            {
+                ReplaceQueryParameterVisitor visitor(query_parameters);
+                visitor.visit(parsed_query);
+            }
+
+            query = serializeAST(*parsed_query);
+        }
+    }
+
     int retries_left = 10;
     while (retries_left)
     {
@@ -2065,6 +2080,41 @@ void ClientBase::initQueryIdFormats()
 }
 
 
+bool ClientBase::addMergeTreeSettings(ASTCreateQuery & ast_create)
+{
+    if (ast_create.attach
+        || !ast_create.storage
+        || !ast_create.storage->isExtendedStorageDefinition()
+        || !ast_create.storage->engine
+        || ast_create.storage->engine->name.find("MergeTree") == std::string::npos)
+        return false;
+
+    auto all_changed = cmd_merge_tree_settings.allChanged();
+    if (all_changed.begin() == all_changed.end())
+        return false;
+
+    if (!ast_create.storage->settings)
+    {
+        auto settings_ast = std::make_shared<ASTSetQuery>();
+        settings_ast->is_standalone = false;
+        ast_create.storage->set(ast_create.storage->settings, settings_ast);
+    }
+
+    auto & storage_settings = *ast_create.storage->settings;
+    bool added_new_setting = false;
+
+    for (const auto & setting : all_changed)
+    {
+        if (!storage_settings.changes.tryGet(setting.getName()))
+        {
+            storage_settings.changes.emplace_back(setting.getName(), setting.getValue());
+            added_new_setting = true;
+        }
+    }
+
+    return added_new_setting;
+}
+
 void ClientBase::runInteractive()
 {
     if (config().has("query_id"))
@@ -2302,6 +2352,30 @@ void ClientBase::parseAndCheckOptions(OptionsDescription & options_description,
         cmd_settings.addProgramOptionsAsMultitokens(options_description.main_description.value());
     else
         cmd_settings.addProgramOptions(options_description.main_description.value());
+
+    if (allow_merge_tree_settings)
+    {
+        /// Add merge tree settings manually, because names of some settings
+        /// may clash. Query settings have higher priority and we just
+        /// skip ambiguous merge tree settings.
+        auto & main_options = options_description.main_description.value();
+
+        NameSet main_option_names;
+        for (const auto & option : main_options.options())
+            main_option_names.insert(option->long_name());
+
+        for (const auto & setting : cmd_merge_tree_settings.all())
+        {
+            if (main_option_names.contains(setting.getName()))
+                continue;
+
+            if (allow_repeated_settings)
+                cmd_merge_tree_settings.addProgramOptionAsMultitoken(main_options, setting);
+            else
+                cmd_merge_tree_settings.addProgramOption(main_options, setting);
+        }
+    }
+
     /// Parse main commandline options.
     auto parser = po::command_line_parser(arguments).options(options_description.main_description.value()).allow_unregistered();
     po::parsed_options parsed = parser.run();

src/Client/ClientBase.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "Common/NamePrompter.h"
+#include <Parsers/ASTCreateQuery.h>
 #include <Common/ProgressIndication.h>
 #include <Common/InterruptListener.h>
 #include <Common/ShellCommand.h>
@@ -14,6 +15,7 @@
 #include <boost/program_options.hpp>
 #include <Storages/StorageFile.h>
 #include <Storages/SelectQueryInfo.h>
+#include <Storages/MergeTree/MergeTreeSettings.h>
 
 
 namespace po = boost::program_options;
@@ -164,6 +166,7 @@ private:
     void updateSuggest(const ASTPtr & ast);
 
     void initQueryIdFormats();
+    bool addMergeTreeSettings(ASTCreateQuery & ast_create);
 
 protected:
     static bool isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context);
@@ -212,6 +215,7 @@ protected:
 
     /// Settings specified via command line args
     Settings cmd_settings;
+    MergeTreeSettings cmd_merge_tree_settings;
 
     /// thread status should be destructed before shared context because it relies on process list.
     std::optional<ThreadStatus> thread_status;
@@ -298,6 +302,7 @@ protected:
     std::vector<HostAndPort> hosts_and_ports{};
 
     bool allow_repeated_settings = false;
+    bool allow_merge_tree_settings = false;
 
     bool cancelled = false;
 

src/Client/Connection.cpp

@@ -686,7 +686,7 @@ void Connection::sendReadTaskResponse(const String & response)
 }
 
 
-void Connection::sendMergeTreeReadTaskResponse(const PartitionReadResponse & response)
+void Connection::sendMergeTreeReadTaskResponse(const ParallelReadResponse & response)
 {
     writeVarUInt(Protocol::Client::MergeTreeReadTaskResponse, *out);
     response.serialize(*out);
@@ -960,8 +960,12 @@ Packet Connection::receivePacket()
             case Protocol::Server::ReadTaskRequest:
                 return res;
 
+            case Protocol::Server::MergeTreeAllRangesAnnounecement:
+                res.announcement = receiveInitialParallelReadAnnounecement();
+                return res;
+
             case Protocol::Server::MergeTreeReadTaskRequest:
-                res.request = receivePartitionReadRequest();
+                res.request = receiveParallelReadRequest();
                 return res;
 
             case Protocol::Server::ProfileEvents:
@@ -1114,13 +1118,20 @@ ProfileInfo Connection::receiveProfileInfo() const
     return profile_info;
 }
 
-PartitionReadRequest Connection::receivePartitionReadRequest() const
+ParallelReadRequest Connection::receiveParallelReadRequest() const
 {
-    PartitionReadRequest request;
+    ParallelReadRequest request;
     request.deserialize(*in);
     return request;
 }
 
+InitialAllRangesAnnouncement Connection::receiveInitialParallelReadAnnounecement() const
+{
+    InitialAllRangesAnnouncement announcement;
+    announcement.deserialize(*in);
+    return announcement;
+}
+
 
 void Connection::throwUnexpectedPacket(UInt64 packet_type, const char * expected) const
 {

src/Client/Connection.h

@@ -110,7 +110,7 @@ public:
 
     void sendData(const Block & block, const String & name/* = "" */, bool scalar/* = false */) override;
 
-    void sendMergeTreeReadTaskResponse(const PartitionReadResponse & response) override;
+    void sendMergeTreeReadTaskResponse(const ParallelReadResponse & response) override;
 
     void sendExternalTablesData(ExternalTablesData & data) override;
 
@@ -265,7 +265,8 @@ private:
     std::vector<String> receiveMultistringMessage(UInt64 msg_type) const;
     std::unique_ptr<Exception> receiveException() const;
     Progress receiveProgress() const;
-    PartitionReadRequest receivePartitionReadRequest() const;
+    ParallelReadRequest receiveParallelReadRequest() const;
+    InitialAllRangesAnnouncement receiveInitialParallelReadAnnounecement() const;
     ProfileInfo receiveProfileInfo() const;
 
     void initInputBuffers();

src/Client/HedgedConnections.h

@@ -94,7 +94,7 @@ public:
         throw Exception(ErrorCodes::LOGICAL_ERROR, "sendReadTaskResponse in not supported with HedgedConnections");
     }
 
-    void sendMergeTreeReadTaskResponse(PartitionReadResponse) override
+    void sendMergeTreeReadTaskResponse(const ParallelReadResponse &) override
     {
         throw Exception(ErrorCodes::LOGICAL_ERROR, "sendMergeTreeReadTaskResponse in not supported with HedgedConnections");
     }

src/Client/IConnections.h

@@ -34,7 +34,7 @@ public:
         bool with_pending_data) = 0;
 
     virtual void sendReadTaskResponse(const String &) = 0;
-    virtual void sendMergeTreeReadTaskResponse(PartitionReadResponse response) = 0;
+    virtual void sendMergeTreeReadTaskResponse(const ParallelReadResponse & response) = 0;
 
     /// Get packet from any replica.
     virtual Packet receivePacket() = 0;
@@ -60,9 +60,9 @@ public:
     /// Get the replica addresses as a string.
     virtual std::string dumpAddresses() const = 0;
 
-
     struct ReplicaInfo
     {
+        bool collaborate_with_initiator{false};
         size_t all_replicas_count{0};
         size_t number_of_current_replica{0};
     };

src/Client/IServerConnection.h

@@ -33,8 +33,10 @@ struct Packet
     Progress progress;
     ProfileInfo profile_info;
     std::vector<UUID> part_uuids;
-    PartitionReadRequest request;
+
-    PartitionReadResponse response;
+    InitialAllRangesAnnouncement announcement;
+    ParallelReadRequest request;
+    ParallelReadResponse response;
 
     Packet() : type(Protocol::Server::Hello) {}
 };
@@ -104,7 +106,7 @@ public:
     /// Send all contents of external (temporary) tables.
     virtual void sendExternalTablesData(ExternalTablesData & data) = 0;
 
-    virtual void sendMergeTreeReadTaskResponse(const PartitionReadResponse & response) = 0;
+    virtual void sendMergeTreeReadTaskResponse(const ParallelReadResponse & response) = 0;
 
     /// Check, if has data to read.
     virtual bool poll(size_t timeout_microseconds) = 0;

src/Client/LocalConnection.cpp

@@ -508,7 +508,7 @@ void LocalConnection::sendExternalTablesData(ExternalTablesData &)
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented");
 }
 
-void LocalConnection::sendMergeTreeReadTaskResponse(const PartitionReadResponse &)
+void LocalConnection::sendMergeTreeReadTaskResponse(const ParallelReadResponse &)
 {
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented");
 }

src/Client/LocalConnection.h

@@ -110,7 +110,7 @@ public:
 
     void sendExternalTablesData(ExternalTablesData &) override;
 
-    void sendMergeTreeReadTaskResponse(const PartitionReadResponse & response) override;
+    void sendMergeTreeReadTaskResponse(const ParallelReadResponse & response) override;
 
     bool poll(size_t timeout_microseconds/* = 0 */) override;
 

src/Client/MultiplexedConnections.cpp

@@ -133,16 +133,11 @@ void MultiplexedConnections::sendQuery(
             modified_settings.group_by_two_level_threshold_bytes = 0;
         }
 
-        bool parallel_reading_from_replicas = settings.max_parallel_replicas > 1
+        if (replica_info)
-            && settings.allow_experimental_parallel_reading_from_replicas
-            /// To avoid trying to coordinate with clickhouse-benchmark,
-            /// since it uses the same code.
-            && client_info.query_kind != ClientInfo::QueryKind::INITIAL_QUERY;
-        if (parallel_reading_from_replicas)
         {
             client_info.collaborate_with_initiator = true;
-            client_info.count_participating_replicas = replica_info.all_replicas_count;
+            client_info.count_participating_replicas = replica_info->all_replicas_count;
-            client_info.number_of_current_replica = replica_info.number_of_current_replica;
+            client_info.number_of_current_replica = replica_info->number_of_current_replica;
         }
     }
 
@@ -199,7 +194,7 @@ void MultiplexedConnections::sendReadTaskResponse(const String & response)
 }
 
 
-void MultiplexedConnections::sendMergeTreeReadTaskResponse(PartitionReadResponse response)
+void MultiplexedConnections::sendMergeTreeReadTaskResponse(const ParallelReadResponse & response)
 {
     std::lock_guard lock(cancel_mutex);
     if (cancelled)
@@ -263,6 +258,7 @@ Packet MultiplexedConnections::drain()
 
         switch (packet.type)
         {
+            case Protocol::Server::MergeTreeAllRangesAnnounecement:
             case Protocol::Server::MergeTreeReadTaskRequest:
             case Protocol::Server::ReadTaskRequest:
             case Protocol::Server::PartUUIDs:
@@ -343,6 +339,7 @@ Packet MultiplexedConnections::receivePacketUnlocked(AsyncCallback async_callbac
 
     switch (packet.type)
     {
+        case Protocol::Server::MergeTreeAllRangesAnnounecement:
         case Protocol::Server::MergeTreeReadTaskRequest:
         case Protocol::Server::ReadTaskRequest:
         case Protocol::Server::PartUUIDs:

src/Client/MultiplexedConnections.h

@@ -42,7 +42,7 @@ public:
         bool with_pending_data) override;
 
     void sendReadTaskResponse(const String &) override;
-    void sendMergeTreeReadTaskResponse(PartitionReadResponse response) override;
+    void sendMergeTreeReadTaskResponse(const ParallelReadResponse & response) override;
 
     Packet receivePacket() override;
 
@@ -104,7 +104,8 @@ private:
     bool sent_query = false;
     bool cancelled = false;
 
-    ReplicaInfo replica_info;
+    /// std::nullopt if parallel reading from replicas is not used
+    std::optional<ReplicaInfo> replica_info;
 
     /// A mutex for the sendCancel function to execute safely
     /// in separate thread.

src/Client/ReplxxLineReader.cpp

@@ -417,6 +417,10 @@ ReplxxLineReader::ReplxxLineReader(
         {
             rx.print("skim failed: %s (consider using Ctrl-T for a regular non-fuzzy reverse search)\n", e.what());
         }
+
+        /// REPAINT before to avoid prompt overlap by the query
+        rx.invoke(Replxx::ACTION::REPAINT, code);
+
         if (!new_query.empty())
             rx.set_state(replxx::Replxx::State(new_query.c_str(), static_cast<int>(new_query.size())));
 

src/Common/Exception.cpp

@@ -559,9 +559,9 @@ bool ExecutionStatus::tryDeserializeText(const std::string & data)
     return true;
 }
 
-ExecutionStatus ExecutionStatus::fromCurrentException(const std::string & start_of_message)
+ExecutionStatus ExecutionStatus::fromCurrentException(const std::string & start_of_message, bool with_stacktrace)
 {
-    String msg = (start_of_message.empty() ? "" : (start_of_message + ": ")) + getCurrentExceptionMessage(false, true);
+    String msg = (start_of_message.empty() ? "" : (start_of_message + ": ")) + getCurrentExceptionMessage(with_stacktrace, true);
     return ExecutionStatus(getCurrentExceptionCode(), msg);
 }
 

src/Common/Exception.h

@@ -242,7 +242,7 @@ struct ExecutionStatus
     explicit ExecutionStatus(int return_code, const std::string & exception_message = "")
     : code(return_code), message(exception_message) {}
 
-    static ExecutionStatus fromCurrentException(const std::string & start_of_message = "");
+    static ExecutionStatus fromCurrentException(const std::string & start_of_message = "", bool with_stacktrace = false);
 
     static ExecutionStatus fromText(const std::string & data);
 

src/Common/OpenTelemetryTraceContext.cpp

@@ -14,76 +14,82 @@ namespace OpenTelemetry
 
 thread_local TracingContextOnThread current_thread_trace_context;
 
-void Span::addAttribute(std::string_view name, UInt64 value)
+bool Span::addAttribute(std::string_view name, UInt64 value) noexcept
 {
     if (!this->isTraceEnabled() || name.empty())
-        return;
+        return false;
 
-    this->attributes.push_back(Tuple{name, toString(value)});
+    return addAttributeImpl(name, toString(value));
 }
 
-void Span::addAttributeIfNotZero(std::string_view name, UInt64 value)
+bool Span::addAttributeIfNotZero(std::string_view name, UInt64 value) noexcept
 {
-    if (value != 0)
+    if (!this->isTraceEnabled() || name.empty() || value == 0)
-        addAttribute(name, value);
+        return false;
+
+    return addAttributeImpl(name, toString(value));
 }
 
-void Span::addAttribute(std::string_view name, std::string_view value)
+bool Span::addAttribute(std::string_view name, std::string_view value) noexcept
 {
     if (!this->isTraceEnabled() || name.empty())
-        return;
+        return false;
 
-    this->attributes.push_back(Tuple{name, value});
+    return addAttributeImpl(name, value);
 }
 
-void Span::addAttributeIfNotEmpty(std::string_view name, std::string_view value)
+bool Span::addAttributeIfNotEmpty(std::string_view name, std::string_view value) noexcept
 {
     if (!this->isTraceEnabled() || name.empty() || value.empty())
-        return;
+        return false;
 
-    this->attributes.push_back(Tuple{name, value});
+    return addAttributeImpl(name, value);
 }
 
-void Span::addAttribute(std::string_view name, std::function<String()> value_supplier)
+bool Span::addAttribute(std::string_view name, std::function<String()> value_supplier) noexcept
 {
-    if (!this->isTraceEnabled() || !value_supplier)
+    if (!this->isTraceEnabled() || name.empty() || !value_supplier)
-        return;
+        return false;
 
-    String value = value_supplier();
+    try
-    if (value.empty())
+    {
-        return;
+        auto value = value_supplier();
-
+        return value.empty() ? false : addAttributeImpl(name, value);
-    this->attributes.push_back(Tuple{name, value});
+    }
+    catch (...)
+    {
+        /// Ignore exception raised by value_supplier
+        return false;
+    }
 }
 
-void Span::addAttribute(const Exception & e) noexcept
+bool Span::addAttribute(const Exception & e) noexcept
 {
     if (!this->isTraceEnabled())
-        return;
+        return false;
 
-    try
+    return addAttributeImpl("clickhouse.exception", getExceptionMessage(e, false));
-    {
-        this->attributes.push_back(Tuple{"clickhouse.exception", getExceptionMessage(e, false)});
-    }
-    catch (...)
-    {
-        /// Ignore exceptions
-    }
 }
 
-void Span::addAttribute(std::exception_ptr e) noexcept
+bool Span::addAttribute(std::exception_ptr e) noexcept
 {
     if (!this->isTraceEnabled() || e == nullptr)
-        return;
+        return false;
 
+    return addAttributeImpl("clickhouse.exception", getExceptionMessage(e, false));
+}
+
+bool Span::addAttributeImpl(std::string_view name, std::string_view value) noexcept
+{
     try
     {
-        this->attributes.push_back(Tuple{"clickhouse.exception", getExceptionMessage(e, false)});
+        this->attributes.push_back(Tuple{name, value});
     }
     catch (...)
     {
-        /// Ignore exceptions
+        return false;
     }
+    return true;
 }
 
 SpanHolder::SpanHolder(std::string_view _operation_name)

src/Common/OpenTelemetryTraceContext.h

@@ -23,21 +23,24 @@ struct Span
     UInt64 finish_time_us = 0;
     Map attributes;
 
-    void addAttribute(std::string_view name, UInt64 value);
+    /// Following methods are declared as noexcept to make sure they're exception safe.
-    void addAttributeIfNotZero(std::string_view name, UInt64 value);
+    /// This is because sometimes they will be called in exception handlers/dtor.
-    void addAttribute(std::string_view name, std::string_view value);
+    /// Returns true if attribute is successfully added and false otherwise.
-    void addAttributeIfNotEmpty(std::string_view name, std::string_view value);
+    bool addAttribute(std::string_view name, UInt64 value) noexcept;
-    void addAttribute(std::string_view name, std::function<String()> value_supplier);
+    bool addAttributeIfNotZero(std::string_view name, UInt64 value) noexcept;
-
+    bool addAttribute(std::string_view name, std::string_view value) noexcept;
-    /// Following two methods are declared as noexcept to make sure they're exception safe
+    bool addAttributeIfNotEmpty(std::string_view name, std::string_view value) noexcept;
-    /// This is because they're usually called in exception handler
+    bool addAttribute(std::string_view name, std::function<String()> value_supplier) noexcept;
-    void addAttribute(const Exception & e) noexcept;
+    bool addAttribute(const Exception & e) noexcept;
-    void addAttribute(std::exception_ptr e) noexcept;
+    bool addAttribute(std::exception_ptr e) noexcept;
 
     bool isTraceEnabled() const
     {
         return trace_id != UUID();
     }
+
+private:
+    bool addAttributeImpl(std::string_view name, std::string_view value) noexcept;
 };
 
 /// See https://www.w3.org/TR/trace-context/ for trace_flags definition

src/Common/ThreadPool.cpp

@@ -3,6 +3,7 @@
 #include <Common/Exception.h>
 #include <Common/getNumberOfPhysicalCPUCores.h>
 #include <Common/OpenTelemetryTraceContext.h>
+#include <Common/noexcept_scope.h>
 
 #include <cassert>
 #include <iostream>
@@ -209,6 +210,7 @@ ThreadPoolImpl<Thread>::~ThreadPoolImpl()
     /// and the destruction order of global variables is unspecified.
 
     finalize();
+    onDestroy();
 }
 
 template <typename Thread>
@@ -227,6 +229,24 @@ void ThreadPoolImpl<Thread>::finalize()
     threads.clear();
 }
 
+template <typename Thread>
+void ThreadPoolImpl<Thread>::addOnDestroyCallback(OnDestroyCallback && callback)
+{
+    std::lock_guard lock(mutex);
+    on_destroy_callbacks.push(std::move(callback));
+}
+
+template <typename Thread>
+void ThreadPoolImpl<Thread>::onDestroy()
+{
+    while (!on_destroy_callbacks.empty())
+    {
+        auto callback = std::move(on_destroy_callbacks.top());
+        on_destroy_callbacks.pop();
+        NOEXCEPT_SCOPE({ callback(); });
+    }
+}
+
 template <typename Thread>
 size_t ThreadPoolImpl<Thread>::active() const
 {

src/Common/ThreadPool.h

@@ -9,6 +9,7 @@
 #include <list>
 #include <optional>
 #include <atomic>
+#include <stack>
 
 #include <boost/heap/priority_queue.hpp>
 
@@ -80,6 +81,16 @@ public:
     void setQueueSize(size_t value);
     size_t getMaxThreads() const;
 
+    /// Adds a callback which is called in destructor after
+    /// joining of all threads. The order of calling callbacks
+    /// is reversed to the order of their addition.
+    /// It may be useful for static thread pools to call
+    /// function after joining of threads because order
+    /// of destructors of global static objects and callbacks
+    /// added by atexit is undefined for different translation units.
+    using OnDestroyCallback = std::function<void()>;
+    void addOnDestroyCallback(OnDestroyCallback && callback);
+
 private:
     mutable std::mutex mutex;
     std::condition_variable job_finished;
@@ -111,6 +122,7 @@ private:
     boost::heap::priority_queue<JobWithPriority> jobs;
     std::list<Thread> threads;
     std::exception_ptr first_exception;
+    std::stack<OnDestroyCallback> on_destroy_callbacks;
 
     template <typename ReturnType>
     ReturnType scheduleImpl(Job job, ssize_t priority, std::optional<uint64_t> wait_microseconds, bool propagate_opentelemetry_tracing_context = true);
@@ -118,6 +130,7 @@ private:
     void worker(typename std::list<Thread>::iterator thread_it);
 
     void finalize();
+    void onDestroy();
 };
 
 
@@ -146,7 +159,8 @@ class GlobalThreadPool : public FreeThreadPool, private boost::noncopyable
             size_t queue_size_, const bool shutdown_on_exception_)
         : FreeThreadPool(max_threads_, max_free_threads_, queue_size_,
             shutdown_on_exception_)
-    {}
+    {
+    }
 
 public:
     static void initialize(size_t max_threads = 10000, size_t max_free_threads = 1000, size_t queue_size = 10000);

src/Compression/CachedCompressedReadBuffer.h

@@ -17,7 +17,7 @@ namespace DB
   * Disadvantages:
   * - in case you need to read a lot of data in a row, but some of them only a part is cached, you have to do seek-and.
   */
-class CachedCompressedReadBuffer : public CompressedReadBufferBase, public ReadBuffer
+class CachedCompressedReadBuffer final : public CompressedReadBufferBase, public ReadBuffer
 {
 private:
     std::function<std::unique_ptr<ReadBufferFromFileBase>()> file_in_creator;

src/Compression/CheckingCompressedReadBuffer.h

@@ -11,7 +11,7 @@ namespace DB
 /** A buffer for reading from a compressed file with just checking checksums of
   * the compressed blocks, without any decompression.
   */
-class CheckingCompressedReadBuffer : public CompressedReadBufferBase, public ReadBuffer
+class CheckingCompressedReadBuffer final : public CompressedReadBufferBase, public ReadBuffer
 {
 protected:
     bool nextImpl() override;

src/Compression/CompressedReadBuffer.h

@@ -8,7 +8,7 @@
 namespace DB
 {
 
-class CompressedReadBuffer : public CompressedReadBufferBase, public BufferWithOwnMemory<ReadBuffer>
+class CompressedReadBuffer final : public CompressedReadBufferBase, public BufferWithOwnMemory<ReadBuffer>
 {
 private:
     size_t size_compressed = 0;

src/Compression/CompressedReadBufferFromFile.h

@@ -14,7 +14,7 @@ class MMappedFileCache;
 
 
 /// Unlike CompressedReadBuffer, it can do seek.
-class CompressedReadBufferFromFile : public CompressedReadBufferBase, public BufferWithOwnMemory<ReadBuffer>
+class CompressedReadBufferFromFile final : public CompressedReadBufferBase, public BufferWithOwnMemory<ReadBuffer>
 {
 private:
       /** At any time, one of two things is true:

src/Compression/CompressedWriteBuffer.h

@@ -13,7 +13,7 @@
 namespace DB
 {
 
-class CompressedWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
+class CompressedWriteBuffer final : public BufferWithOwnMemory<WriteBuffer>
 {
 public:
     explicit CompressedWriteBuffer(

src/Compression/CompressionCodecDeflateQpl.h

@@ -87,7 +87,7 @@ private:
     Poco::Logger * log;
 };
 
-class CompressionCodecDeflateQpl : public ICompressionCodec
+class CompressionCodecDeflateQpl final : public ICompressionCodec
 {
 public:
     CompressionCodecDeflateQpl();

src/Compression/CompressionCodecEncrypted.cpp

@@ -11,10 +11,14 @@
 
 // This depends on BoringSSL-specific API, notably <openssl/aead.h>.
 #if USE_SSL
-#include <openssl/digest.h>
+#    include <openssl/err.h>
-#include <openssl/err.h>
+#    include <boost/algorithm/hex.hpp>
-#include <boost/algorithm/hex.hpp>
+#    if USE_BORINGSSL
-#include <openssl/aead.h>
+#        include <openssl/digest.h>
+#        include <openssl/aead.h>
+#    else
+#        include <openssl/evp.h>
+#    endif
 #endif
 
 // Common part for both parts (with SSL and without)
@@ -87,23 +91,6 @@ constexpr size_t nonce_max_size    = 13;   /// Nonce size and one byte to show i
 constexpr size_t actual_nonce_size = 12;   /// Nonce actual size
 const String empty_nonce = {"\0\0\0\0\0\0\0\0\0\0\0\0", actual_nonce_size};
 
-/// Get encryption/decryption algorithms.
-auto getMethod(EncryptionMethod Method)
-{
-    if (Method == AES_128_GCM_SIV)
-    {
-        return EVP_aead_aes_128_gcm_siv;
-    }
-    else if (Method == AES_256_GCM_SIV)
-    {
-        return EVP_aead_aes_256_gcm_siv;
-    }
-    else
-    {
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong encryption Method. Got {}", getMethodName(Method));
-    }
-}
-
 /// Find out key size for each algorithm
 UInt64 methodKeySize(EncryptionMethod Method)
 {
@@ -128,6 +115,24 @@ std::string lastErrorString()
     return std::string(buffer.data());
 }
 
+#if USE_BORINGSSL
+/// Get encryption/decryption algorithms.
+auto getMethod(EncryptionMethod Method)
+{
+    if (Method == AES_128_GCM_SIV)
+    {
+        return EVP_aead_aes_128_gcm_siv;
+    }
+    else if (Method == AES_256_GCM_SIV)
+    {
+        return EVP_aead_aes_256_gcm_siv;
+    }
+    else
+    {
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong encryption Method. Got {}", getMethodName(Method));
+    }
+}
+
 /// Encrypt plaintext with particular algorithm and put result into ciphertext_and_tag.
 /// This function get key and nonce and encrypt text with their help.
 /// If something went wrong (can't init context or can't encrypt data) it throws exception.
@@ -186,6 +191,160 @@ size_t decrypt(std::string_view ciphertext, char * plaintext, EncryptionMethod m
 
     return out_len;
 }
+#else
+/// Get encryption/decryption algorithms.
+auto getMethod(EncryptionMethod Method)
+{
+    if (Method == AES_128_GCM_SIV)
+    {
+        return EVP_aes_128_gcm;
+    }
+    else if (Method == AES_256_GCM_SIV)
+    {
+        return EVP_aes_256_gcm;
+    }
+    else
+    {
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong encryption Method. Got {}", getMethodName(Method));
+    }
+}
+
+/// Encrypt plaintext with particular algorithm and put result into ciphertext_and_tag.
+/// This function get key and nonce and encrypt text with their help.
+/// If something went wrong (can't init context or can't encrypt data) it throws exception.
+/// It returns length of encrypted text.
+size_t encrypt(std::string_view plaintext, char * ciphertext_and_tag, EncryptionMethod method, const String & key, const String & nonce)
+{
+    int out_len;
+    int ciphertext_len;
+    EVP_CIPHER_CTX *encrypt_ctx;
+
+    if (!(encrypt_ctx = EVP_CIPHER_CTX_new()))
+        throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
+
+    try
+    {
+        const int ok_cryptinit = EVP_EncryptInit_ex(encrypt_ctx,
+                                                    getMethod(method)(),
+                                                    nullptr, nullptr, nullptr);
+        if (!ok_cryptinit)
+            throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
+
+        const int ok_cipherctrl = EVP_CIPHER_CTX_ctrl(encrypt_ctx,
+                                                    EVP_CTRL_GCM_SET_IVLEN,
+                                                    static_cast<int32_t>(nonce.size()),
+                                                    nullptr);
+        if (!ok_cipherctrl)
+            throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
+
+        const int ok_nonceinit = EVP_EncryptInit_ex(encrypt_ctx, nullptr, nullptr,
+                                                    reinterpret_cast<const uint8_t*>(key.data()),
+                                                    reinterpret_cast<const uint8_t *>(nonce.data()));
+        if (!ok_nonceinit)
+            throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
+
+        const int ok_encryptupdate = EVP_EncryptUpdate(encrypt_ctx,
+                                                    reinterpret_cast<uint8_t *>(ciphertext_and_tag),
+                                                    &out_len,
+                                                    reinterpret_cast<const uint8_t *>(plaintext.data()),
+                                                    static_cast<int32_t>(plaintext.size()));
+        ciphertext_len = out_len;
+        if (!ok_encryptupdate)
+            throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
+
+        const int ok_encryptfinal = EVP_EncryptFinal_ex(encrypt_ctx,
+                                                        reinterpret_cast<uint8_t *>(ciphertext_and_tag) + out_len,
+                                                        reinterpret_cast<int32_t *>(&out_len));
+        ciphertext_len += out_len;
+        if (!ok_encryptfinal)
+            throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
+
+        /* Get the tag */
+        const int ok_tag = EVP_CIPHER_CTX_ctrl(encrypt_ctx,
+                                            EVP_CTRL_GCM_GET_TAG,
+                                            tag_size,
+                                            reinterpret_cast<uint8_t *>(ciphertext_and_tag) + plaintext.size());
+
+        if (!ok_tag)
+            throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
+    }
+    catch (...)
+    {
+        EVP_CIPHER_CTX_free(encrypt_ctx);
+        throw;
+    }
+    EVP_CIPHER_CTX_free(encrypt_ctx);
+    return ciphertext_len + tag_size;
+}
+
+/// Encrypt plaintext with particular algorithm and put result into ciphertext_and_tag.
+/// This function get key and nonce and encrypt text with their help.
+/// If something went wrong (can't init context or can't encrypt data) it throws exception.
+/// It returns length of encrypted text.
+size_t decrypt(std::string_view ciphertext, char * plaintext, EncryptionMethod method, const String & key, const String & nonce)
+{
+
+    int out_len;
+    int plaintext_len;
+    EVP_CIPHER_CTX *decrypt_ctx;
+
+    if (!(decrypt_ctx = EVP_CIPHER_CTX_new()))
+        throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
+
+    try
+    {
+        const int ok_cryptinit = EVP_DecryptInit_ex(decrypt_ctx,
+                                                    getMethod(method)(),
+                                                    nullptr, nullptr, nullptr);
+        if (!ok_cryptinit)
+            throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
+
+        const int ok_cipherctrl = EVP_CIPHER_CTX_ctrl(decrypt_ctx,
+                                                    EVP_CTRL_GCM_SET_IVLEN,
+                                                    static_cast<int32_t>(nonce.size()), nullptr);
+        if (!ok_cipherctrl)
+            throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
+
+        const int ok_nonceinit = EVP_DecryptInit_ex(decrypt_ctx, nullptr, nullptr,
+                                                    reinterpret_cast<const uint8_t*>(key.data()),
+                                                    reinterpret_cast<const uint8_t *>(nonce.data()));
+        if (!ok_nonceinit)
+            throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
+
+        const int ok_decryptudpate = EVP_DecryptUpdate(decrypt_ctx,
+                                                    reinterpret_cast<uint8_t *>(plaintext),
+                                                    reinterpret_cast<int32_t *>(&out_len),
+                                                    reinterpret_cast<const uint8_t *>(ciphertext.data()),
+                                                    static_cast<int32_t>(ciphertext.size()) - tag_size);
+        plaintext_len = out_len;
+
+        if (!ok_decryptudpate)
+            throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
+
+        const int ok_tag = EVP_CIPHER_CTX_ctrl(decrypt_ctx,
+                                            EVP_CTRL_GCM_SET_TAG,
+                                            tag_size,
+                                            reinterpret_cast<uint8_t *>(const_cast<char *>(ciphertext.data())) + ciphertext.size() - tag_size);
+        if (!ok_tag)
+            throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
+
+        const int ok_decryptfinal = EVP_DecryptFinal_ex(decrypt_ctx,
+                                                        reinterpret_cast<uint8_t *>(plaintext) + out_len,
+                                                        reinterpret_cast<int32_t *>(&out_len));
+
+        if (!ok_decryptfinal)
+            throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
+    }
+    catch (...)
+    {
+        EVP_CIPHER_CTX_free(decrypt_ctx);
+        throw;
+    }
+    EVP_CIPHER_CTX_free(decrypt_ctx);
+
+    return plaintext_len + out_len;
+}
+#endif
 
 /// Register codec in factory
 void registerEncryptionCodec(CompressionCodecFactory & factory, EncryptionMethod Method)

src/Compression/CompressionCodecEncrypted.h

@@ -44,7 +44,7 @@ enum EncryptionMethod
     * as otherwise our engines like ReplicatedMergeTree cannot
     * deduplicate data blocks.
     */
-class CompressionCodecEncrypted : public ICompressionCodec
+class CompressionCodecEncrypted final : public ICompressionCodec
 {
 public:
     /** If a key is available, the server is supposed to

src/Compression/CompressionCodecNone.h

@@ -8,7 +8,7 @@
 namespace DB
 {
 
-class CompressionCodecNone : public ICompressionCodec
+class CompressionCodecNone final : public ICompressionCodec
 {
 public:
     CompressionCodecNone();

src/Compression/CompressionFactory.cpp

@@ -178,9 +178,7 @@ void registerCodecDelta(CompressionCodecFactory & factory);
 void registerCodecT64(CompressionCodecFactory & factory);
 void registerCodecDoubleDelta(CompressionCodecFactory & factory);
 void registerCodecGorilla(CompressionCodecFactory & factory);
-#if USE_BORINGSSL
 void registerCodecEncrypted(CompressionCodecFactory & factory);
-#endif
 void registerCodecFPC(CompressionCodecFactory & factory);
 
 #endif
@@ -197,9 +195,7 @@ CompressionCodecFactory::CompressionCodecFactory()
     registerCodecT64(*this);
     registerCodecDoubleDelta(*this);
     registerCodecGorilla(*this);
-#if USE_BORINGSSL
     registerCodecEncrypted(*this);
-#endif
     registerCodecFPC(*this);
     #ifdef ENABLE_QPL_COMPRESSION
         registerCodecDeflateQpl(*this);

src/Coordination/KeeperSnapshotManagerS3.cpp

@@ -17,10 +17,7 @@
 #include <Common/Macros.h>
 
 #include <aws/core/auth/AWSCredentials.h>
-#include <aws/s3/S3Client.h>
 #include <aws/s3/S3Errors.h>
-#include <aws/s3/model/HeadObjectRequest.h>
-#include <aws/s3/model/DeleteObjectRequest.h>
 
 #include <filesystem>
 
@@ -31,7 +28,7 @@ namespace DB
 
 struct KeeperSnapshotManagerS3::S3Configuration
 {
-    S3Configuration(S3::URI uri_, S3::AuthSettings auth_settings_, std::shared_ptr<const Aws::S3::S3Client> client_)
+    S3Configuration(S3::URI uri_, S3::AuthSettings auth_settings_, std::shared_ptr<const S3::Client> client_)
         : uri(std::move(uri_))
         , auth_settings(std::move(auth_settings_))
         , client(std::move(client_))
@@ -39,7 +36,7 @@ struct KeeperSnapshotManagerS3::S3Configuration
 
     S3::URI uri;
     S3::AuthSettings auth_settings;
-    std::shared_ptr<const Aws::S3::S3Client> client;
+    std::shared_ptr<const S3::Client> client;
 };
 
 KeeperSnapshotManagerS3::KeeperSnapshotManagerS3()
@@ -202,7 +199,7 @@ void KeeperSnapshotManagerS3::uploadSnapshotImpl(const std::string & snapshot_pa
             LOG_INFO(log, "Removing lock file");
             try
             {
-                Aws::S3::Model::DeleteObjectRequest delete_request;
+                S3::DeleteObjectRequest delete_request;
                 delete_request.SetBucket(s3_client->uri.bucket);
                 delete_request.SetKey(lock_file);
                 auto delete_outcome = s3_client->client->DeleteObject(delete_request);

src/Core/BaseSettings.h

@@ -5,10 +5,17 @@
 #include <base/range.h>
 #include <boost/blank.hpp>
 #include <unordered_map>
+#include <boost/program_options/options_description.hpp>
 
 
+namespace boost::program_options
+{
+    class options_description;
+}
+
 namespace DB
 {
+
 class ReadBuffer;
 class WriteBuffer;
 
@@ -19,7 +26,6 @@ enum class SettingsWriteFormat
     DEFAULT = STRINGS_WITH_FLAGS,
 };
 
-
 /** Template class to define collections of settings.
   * Example of usage:
   *
@@ -119,6 +125,18 @@ public:
         std::conditional_t<Traits::allow_custom_settings, const CustomSettingMap::mapped_type*, boost::blank> custom_setting;
     };
 
+    /// Adds program options to set the settings from a command line.
+    /// (Don't forget to call notify() on the `variables_map` after parsing it!)
+    void addProgramOptions(boost::program_options::options_description & options);
+
+    /// Adds program options as to set the settings from a command line.
+    /// Allows to set one setting multiple times, the last value will be used.
+    /// (Don't forget to call notify() on the `variables_map` after parsing it!)
+    void addProgramOptionsAsMultitokens(boost::program_options::options_description & options);
+
+    void addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field);
+    void addProgramOptionAsMultitoken(boost::program_options::options_description & options, const SettingFieldRef & field);
+
     enum SkipFlags
     {
         SKIP_NONE = 0,
@@ -518,6 +536,38 @@ String BaseSettings<TTraits>::toString() const
     return res;
 }
 
+template <typename TTraits>
+void BaseSettings<TTraits>::addProgramOptions(boost::program_options::options_description & options)
+{
+    for (const auto & field : all())
+        addProgramOption(options, field);
+}
+
+template <typename TTraits>
+void BaseSettings<TTraits>::addProgramOptionsAsMultitokens(boost::program_options::options_description & options)
+{
+    for (const auto & field : all())
+        addProgramOptionAsMultitoken(options, field);
+}
+
+template <typename TTraits>
+void BaseSettings<TTraits>::addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field)
+{
+    const std::string_view name = field.getName();
+    auto on_program_option = boost::function1<void, const std::string &>([this, name](const std::string & value) { set(name, value); });
+    options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
+        name.data(), boost::program_options::value<std::string>()->composing()->notifier(on_program_option), field.getDescription())));
+}
+
+template <typename TTraits>
+void BaseSettings<TTraits>::addProgramOptionAsMultitoken(boost::program_options::options_description & options, const SettingFieldRef & field)
+{
+    const std::string_view name = field.getName();
+    auto on_program_option = boost::function1<void, const Strings &>([this, name](const Strings & values) { set(name, values.back()); });
+    options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
+        name.data(), boost::program_options::value<Strings>()->multitoken()->composing()->notifier(on_program_option), field.getDescription())));
+}
+
 template <typename TTraits>
 bool operator==(const BaseSettings<TTraits> & left, const BaseSettings<TTraits> & right)
 {

src/Core/Protocol.h

@@ -81,7 +81,8 @@ namespace Protocol
                                             /// This is such an inverted logic, where server sends requests
                                             /// And client returns back response
             ProfileEvents = 14,             /// Packet with profile events from server.
-            MergeTreeReadTaskRequest = 15,  /// Request from a MergeTree replica to a coordinator
+            MergeTreeAllRangesAnnounecement = 15,
+            MergeTreeReadTaskRequest = 16,  /// Request from a MergeTree replica to a coordinator
             MAX = MergeTreeReadTaskRequest,
 
         };
@@ -108,6 +109,7 @@ namespace Protocol
                 "PartUUIDs",
                 "ReadTaskRequest",
                 "ProfileEvents",
+                "MergeTreeAllRangesAnnounecement",
                 "MergeTreeReadTaskRequest",
             };
             return packet <= MAX

src/Core/ProtocolDefines.h

@@ -33,6 +33,8 @@
 #define DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION 1
 #define DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS 54453
 
+#define DBMS_MERGE_TREE_PART_INFO_VERSION 1
+
 /// Minimum revision supporting interserver secret.
 #define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET 54441
 

src/Core/Settings.cpp

@@ -6,7 +6,6 @@
 #include <Columns/ColumnMap.h>
 #include <Common/typeid_cast.h>
 #include <cstring>
-#include <boost/program_options/options_description.hpp>
 
 namespace DB
 {
@@ -82,38 +81,6 @@ void Settings::dumpToMapColumn(IColumn * column, bool changed_only)
     offsets.push_back(offsets.back() + size);
 }
 
-void Settings::addProgramOptions(boost::program_options::options_description & options)
-{
-    for (const auto & field : all())
-    {
-        addProgramOption(options, field);
-    }
-}
-
-void Settings::addProgramOptionsAsMultitokens(boost::program_options::options_description & options)
-{
-    for (const auto & field : all())
-    {
-        addProgramOptionAsMultitoken(options, field);
-    }
-}
-
-void Settings::addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field)
-{
-    const std::string_view name = field.getName();
-    auto on_program_option = boost::function1<void, const std::string &>([this, name](const std::string & value) { set(name, value); });
-    options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
-        name.data(), boost::program_options::value<std::string>()->composing()->notifier(on_program_option), field.getDescription())));
-}
-
-void Settings::addProgramOptionAsMultitoken(boost::program_options::options_description & options, const SettingFieldRef & field)
-{
-    const std::string_view name = field.getName();
-    auto on_program_option = boost::function1<void, const Strings &>([this, name](const Strings & values) { set(name, values.back()); });
-    options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
-        name.data(), boost::program_options::value<Strings>()->multitoken()->composing()->notifier(on_program_option), field.getDescription())));
-}
-
 void Settings::checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path)
 {
     if (config.getBool("skip_check_for_incorrect_settings", false))

src/Core/Settings.h

@@ -13,12 +13,6 @@ namespace Poco::Util
     class AbstractConfiguration;
 }
 
-namespace boost::program_options
-{
-    class options_description;
-}
-
-
 namespace DB
 {
 class IColumn;
@@ -151,7 +145,9 @@ class IColumn;
     M(UInt64, parallel_replicas_count, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the number of parallel replicas participating in query processing.", 0) \
     M(UInt64, parallel_replica_offset, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the index of the replica participating in query processing among parallel replicas.", 0) \
     \
+    M(String, cluster_for_parallel_replicas, "default", "Cluster for a shard in which current server is located", 0) \
     M(Bool, allow_experimental_parallel_reading_from_replicas, false, "If true, ClickHouse will send a SELECT query to all replicas of a table. It will work for any kind on MergeTree table.", 0) \
+    M(Float, parallel_replicas_single_task_marks_count_multiplier, 2, "A multiplier which will be added during calculation for minimal number of marks to retrieve from coordinator. This will be applied only for remote replicas.", 0) \
     \
     M(Bool, skip_unavailable_shards, false, "If true, ClickHouse silently skips unavailable shards and nodes unresolvable through DNS. Shard is marked as unavailable when none of the replicas can be reached.", 0) \
     \
@@ -603,7 +599,7 @@ class IColumn;
     M(ShortCircuitFunctionEvaluation, short_circuit_function_evaluation, ShortCircuitFunctionEvaluation::ENABLE, "Setting for short-circuit function evaluation configuration. Possible values: 'enable' - use short-circuit function evaluation for functions that are suitable for it, 'disable' - disable short-circuit function evaluation, 'force_enable' - use short-circuit function evaluation for all functions.", 0) \
     \
     M(LocalFSReadMethod, storage_file_read_method, LocalFSReadMethod::mmap, "Method of reading data from storage file, one of: read, pread, mmap.", 0) \
-    M(String, local_filesystem_read_method, "pread_threadpool", "Method of reading data from local filesystem, one of: read, pread, mmap, io_uring, pread_threadpool.", 0) \
+    M(String, local_filesystem_read_method, "pread_threadpool", "Method of reading data from local filesystem, one of: read, pread, mmap, io_uring, pread_threadpool. The 'io_uring' method is experimental and does not work for Log, TinyLog, StripeLog, File, Set and Join, and other tables with append-able files in presence of concurrent reads and writes.", 0) \
     M(String, remote_filesystem_read_method, "threadpool", "Method of reading data from remote filesystem, one of: read, threadpool.", 0) \
     M(Bool, local_filesystem_read_prefetch, false, "Should use prefetching when reading data from local filesystem.", 0) \
     M(Bool, remote_filesystem_read_prefetch, true, "Should use prefetching when reading data from remote filesystem.", 0) \
@@ -774,6 +770,7 @@ class IColumn;
     M(Bool, input_format_tsv_detect_header, true, "Automatically detect header with names and types in TSV format", 0) \
     M(Bool, input_format_custom_detect_header, true, "Automatically detect header with names and types in CustomSeparated format", 0) \
     M(Bool, input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference, false, "Skip columns with unsupported types while schema inference for format Parquet", 0) \
+    M(UInt64, input_format_parquet_max_block_size, 8192, "Max block size for parquet reader.", 0) \
     M(Bool, input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference, false, "Skip fields with unsupported types while schema inference for format Protobuf", 0) \
     M(Bool, input_format_capn_proto_skip_fields_with_unsupported_types_in_schema_inference, false, "Skip columns with unsupported types while schema inference for format CapnProto", 0) \
     M(Bool, input_format_orc_skip_columns_with_unsupported_types_in_schema_inference, false, "Skip columns with unsupported types while schema inference for format ORC", 0) \
@@ -923,25 +920,12 @@ struct Settings : public BaseSettings<SettingsTraits>, public IHints<2, Settings
     /// Dumps profile events to column of type Map(String, String)
     void dumpToMapColumn(IColumn * column, bool changed_only = true);
 
-    /// Adds program options to set the settings from a command line.
-    /// (Don't forget to call notify() on the `variables_map` after parsing it!)
-    void addProgramOptions(boost::program_options::options_description & options);
-
-    /// Adds program options as to set the settings from a command line.
-    /// Allows to set one setting multiple times, the last value will be used.
-    /// (Don't forget to call notify() on the `variables_map` after parsing it!)
-    void addProgramOptionsAsMultitokens(boost::program_options::options_description & options);
-
     /// Check that there is no user-level settings at the top level in config.
     /// This is a common source of mistake (user don't know where to write user-level setting).
     static void checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path);
 
     std::vector<String> getAllRegisteredNames() const override;
 
-    void addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field);
-
-    void addProgramOptionAsMultitoken(boost::program_options::options_description & options, const SettingFieldRef & field);
-
     void set(std::string_view name, const Field & value) override;
 
     void setDefaultValue(const String & name) { resetToDefault(name); }

src/Disks/IDisk.h

@@ -28,6 +28,7 @@ namespace Poco
 {
     namespace Util
     {
+        /// NOLINTNEXTLINE(cppcoreguidelines-virtual-class-destructor)
         class AbstractConfiguration;
     }
 }

src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp

@@ -21,10 +21,6 @@
 #include <Interpreters/threadPoolCallbackRunner.h>
 #include <Disks/ObjectStorages/S3/diskSettings.h>
 
-#include <aws/s3/model/ListObjectsV2Request.h>
-#include <aws/s3/model/DeleteObjectRequest.h>
-#include <aws/s3/model/DeleteObjectsRequest.h>
-
 #include <Common/getRandomASCIIString.h>
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/logger_useful.h>
@@ -213,7 +209,7 @@ void S3ObjectStorage::findAllFiles(const std::string & path, RelativePathsWithSi
     auto settings_ptr = s3_settings.get();
     auto client_ptr = client.get();
 
-    Aws::S3::Model::ListObjectsV2Request request;
+    S3::ListObjectsV2Request request;
     request.SetBucket(bucket);
     request.SetPrefix(path);
     if (max_keys)
@@ -257,7 +253,7 @@ void S3ObjectStorage::getDirectoryContents(const std::string & path,
     auto settings_ptr = s3_settings.get();
     auto client_ptr = client.get();
 
-    Aws::S3::Model::ListObjectsV2Request request;
+    S3::ListObjectsV2Request request;
     request.SetBucket(bucket);
     /// NOTE: if you do "ls /foo" instead of "ls /foo/" over S3 with this API
     /// it will return only "/foo" itself without any underlying nodes.
@@ -304,7 +300,7 @@ void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exis
 
     ProfileEvents::increment(ProfileEvents::S3DeleteObjects);
     ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects);
-    Aws::S3::Model::DeleteObjectRequest request;
+    S3::DeleteObjectRequest request;
     request.SetBucket(bucket);
     request.SetKey(object.absolute_path);
     auto outcome = client_ptr->DeleteObject(request);
@@ -352,7 +348,7 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e
 
             ProfileEvents::increment(ProfileEvents::S3DeleteObjects);
             ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects);
-            Aws::S3::Model::DeleteObjectsRequest request;
+            S3::DeleteObjectsRequest request;
             request.SetBucket(bucket);
             request.SetDelete(delkeys);
             auto outcome = client_ptr->DeleteObjects(request);
@@ -435,7 +431,7 @@ void S3ObjectStorage::setNewSettings(std::unique_ptr<S3ObjectStorageSettings> &&
     s3_settings.set(std::move(s3_settings_));
 }
 
-void S3ObjectStorage::setNewClient(std::unique_ptr<Aws::S3::S3Client> && client_)
+void S3ObjectStorage::setNewClient(std::unique_ptr<S3::Client> && client_)
 {
     client.set(std::move(client_));
 }
@@ -447,7 +443,7 @@ void S3ObjectStorage::shutdown()
     /// If S3 request is failed and the method below is executed S3 client immediately returns the last failed S3 request outcome.
     /// If S3 is healthy nothing wrong will be happened and S3 requests will be processed in a regular way without errors.
     /// This should significantly speed up shutdown process if S3 is unhealthy.
-    const_cast<Aws::S3::S3Client &>(*client_ptr).DisableRequestProcessing();
+    const_cast<S3::Client &>(*client_ptr).DisableRequestProcessing();
 }
 
 void S3ObjectStorage::startup()
@@ -455,7 +451,7 @@ void S3ObjectStorage::startup()
     auto client_ptr = client.get();
 
     /// Need to be enabled if it was disabled during shutdown() call.
-    const_cast<Aws::S3::S3Client &>(*client_ptr).EnableRequestProcessing();
+    const_cast<S3::Client &>(*client_ptr).EnableRequestProcessing();
 }
 
 void S3ObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context)

src/Disks/ObjectStorages/S3/S3ObjectStorage.h

@@ -7,7 +7,6 @@
 #include <Disks/ObjectStorages/IObjectStorage.h>
 #include <Disks/ObjectStorages/S3/S3Capabilities.h>
 #include <memory>
-#include <aws/s3/S3Client.h>
 #include <Storages/StorageS3Settings.h>
 #include <Common/MultiVersion.h>
 #include <Common/logger_useful.h>
@@ -46,7 +45,7 @@ private:
 
     S3ObjectStorage(
         const char * logger_name,
-        std::unique_ptr<Aws::S3::S3Client> && client_,
+        std::unique_ptr<S3::Client> && client_,
         std::unique_ptr<S3ObjectStorageSettings> && s3_settings_,
         String version_id_,
         const S3Capabilities & s3_capabilities_,
@@ -68,7 +67,7 @@ private:
 
 public:
     template <class ...Args>
-    S3ObjectStorage(std::unique_ptr<Aws::S3::S3Client> && client_, Args && ...args)
+    explicit S3ObjectStorage(std::unique_ptr<S3::Client> && client_, Args && ...args)
         : S3ObjectStorage("S3ObjectStorage", std::move(client_), std::forward<Args>(args)...)
     {
     }
@@ -163,14 +162,14 @@ public:
 private:
     void setNewSettings(std::unique_ptr<S3ObjectStorageSettings> && s3_settings_);
 
-    void setNewClient(std::unique_ptr<Aws::S3::S3Client> && client_);
+    void setNewClient(std::unique_ptr<S3::Client> && client_);
 
     void removeObjectImpl(const StoredObject & object, bool if_exists);
     void removeObjectsImpl(const StoredObjects & objects, bool if_exists);
 
     std::string bucket;
 
-    MultiVersion<Aws::S3::S3Client> client;
+    MultiVersion<S3::Client> client;
     MultiVersion<S3ObjectStorageSettings> s3_settings;
     S3Capabilities s3_capabilities;
 
@@ -191,7 +190,7 @@ public:
     std::string getName() const override { return "S3PlainObjectStorage"; }
 
     template <class ...Args>
-    S3PlainObjectStorage(Args && ...args)
+    explicit S3PlainObjectStorage(Args && ...args)
         : S3ObjectStorage("S3PlainObjectStorage", std::forward<Args>(args)...)
     {
         data_source_description.type = DataSourceType::S3_Plain;

src/Disks/ObjectStorages/S3/diskSettings.cpp

@@ -107,7 +107,7 @@ std::shared_ptr<S3::ProxyConfiguration> getProxyConfiguration(const String & pre
 }
 
 
-std::unique_ptr<Aws::S3::S3Client> getClient(
+std::unique_ptr<S3::Client> getClient(
     const Poco::Util::AbstractConfiguration & config,
     const String & config_prefix,
     ContextPtr context,

src/Disks/ObjectStorages/S3/diskSettings.h

@@ -7,11 +7,13 @@
 #include <Poco/Util/AbstractConfiguration.h>
 #include <Interpreters/Context_fwd.h>
 
+#include <IO/S3/Client.h>
+
 namespace Aws
 {
 namespace S3
 {
-class S3Client;
+class Client;
 }
 }
 
@@ -22,7 +24,7 @@ struct S3ObjectStorageSettings;
 
 std::unique_ptr<S3ObjectStorageSettings> getSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context);
 
-std::unique_ptr<Aws::S3::S3Client> getClient(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context, const S3ObjectStorageSettings & settings);
+std::unique_ptr<S3::Client> getClient(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context, const S3ObjectStorageSettings & settings);
 
 }
 

src/Formats/FormatFactory.cpp

@@ -116,6 +116,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings)
     format_settings.parquet.allow_missing_columns = settings.input_format_parquet_allow_missing_columns;
     format_settings.parquet.skip_columns_with_unsupported_types_in_schema_inference = settings.input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference;
     format_settings.parquet.output_string_as_string = settings.output_format_parquet_string_as_string;
+    format_settings.parquet.max_block_size = settings.input_format_parquet_max_block_size;
     format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8;
     format_settings.pretty.color = settings.output_format_pretty_color;
     format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width;

src/Formats/FormatSettings.h

@@ -183,6 +183,7 @@ struct FormatSettings
         bool case_insensitive_column_matching = false;
         std::unordered_set<int> skip_row_groups = {};
         bool output_string_as_string = false;
+        UInt64 max_block_size = 8192;
     } parquet;
 
     struct Pretty

src/Functions/FunctionsConversion.h

@@ -377,7 +377,7 @@ struct ToDateTransform32Or64
     static NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone)
     {
         // since converting to Date, no need in values outside of default LUT range.
-        return (from <= DATE_LUT_MAX_DAY_NUM)
+        return (from < DATE_LUT_MAX_DAY_NUM)
             ? from
             : time_zone.toDayNum(std::min(time_t(from), time_t(0xFFFFFFFF)));
     }
@@ -394,7 +394,7 @@ struct ToDateTransform32Or64Signed
         /// The function should be monotonic (better for query optimizations), so we saturate instead of overflow.
         if (from < 0)
             return 0;
-        return (from <= DATE_LUT_MAX_DAY_NUM)
+        return (from < DATE_LUT_MAX_DAY_NUM)
             ? static_cast<ToType>(from)
             : time_zone.toDayNum(std::min(time_t(from), time_t(0xFFFFFFFF)));
     }

src/Functions/tupleElement.cpp

@@ -56,10 +56,9 @@ public:
         return true;
     }
 
-    ColumnNumbers getArgumentsThatAreAlwaysConstant() const override
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
-    {
+
-        return {1};
+    bool useDefaultImplementationForNulls() const override { return false; }
-    }
 
     bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
 

src/IO/ReadBufferFromS3.cpp

@@ -7,9 +7,7 @@
 #include <IO/ReadBufferFromS3.h>
 #include <IO/S3/getObjectInfo.h>
 
-#include <aws/s3/S3Client.h>
+#include <IO/S3/Requests.h>
-#include <aws/s3/model/GetObjectRequest.h>
-#include <aws/s3/model/HeadObjectRequest.h>
 
 #include <Common/Stopwatch.h>
 #include <Common/Throttler.h>
@@ -44,7 +42,7 @@ namespace ErrorCodes
 
 
 ReadBufferFromS3::ReadBufferFromS3(
-    std::shared_ptr<const Aws::S3::S3Client> client_ptr_,
+    std::shared_ptr<const S3::Client> client_ptr_,
     const String & bucket_,
     const String & key_,
     const String & version_id_,
@@ -281,7 +279,7 @@ SeekableReadBuffer::Range ReadBufferFromS3::getRemainingReadRange() const
 
 std::unique_ptr<ReadBuffer> ReadBufferFromS3::initialize()
 {
-    Aws::S3::Model::GetObjectRequest req;
+    S3::GetObjectRequest req;
     req.SetBucket(bucket);
     req.SetKey(key);
     if (!version_id.empty())

src/IO/ReadBufferFromS3.h

@@ -19,7 +19,7 @@
 
 namespace Aws::S3
 {
-class S3Client;
+class Client;
 }
 
 namespace DB
@@ -30,7 +30,7 @@ namespace DB
 class ReadBufferFromS3 : public ReadBufferFromFileBase
 {
 private:
-    std::shared_ptr<const Aws::S3::S3Client> client_ptr;
+    std::shared_ptr<const S3::Client> client_ptr;
     String bucket;
     String key;
     String version_id;
@@ -49,7 +49,7 @@ private:
 
 public:
     ReadBufferFromS3(
-        std::shared_ptr<const Aws::S3::S3Client> client_ptr_,
+        std::shared_ptr<const S3::Client> client_ptr_,
         const String & bucket_,
         const String & key_,
         const String & version_id_,
@@ -95,7 +95,7 @@ class ReadBufferS3Factory : public ParallelReadBuffer::ReadBufferFactory, public
 {
 public:
     explicit ReadBufferS3Factory(
-        std::shared_ptr<const Aws::S3::S3Client> client_ptr_,
+        std::shared_ptr<const S3::Client> client_ptr_,
         const String & bucket_,
         const String & key_,
         const String & version_id_,
@@ -126,7 +126,7 @@ public:
     String getFileName() const override { return bucket + "/" + key; }
 
 private:
-    std::shared_ptr<const Aws::S3::S3Client> client_ptr;
+    std::shared_ptr<const S3::Client> client_ptr;
     const String bucket;
     const String key;
     const String version_id;

src/IO/S3/Client.cpp

@@ -0,0 +1,400 @@
+#include <IO/S3/Client.h>
+
+#if USE_AWS_S3
+
+#include <aws/core/client/DefaultRetryStrategy.h>
+#include <aws/s3/model/HeadBucketRequest.h>
+#include <aws/s3/model/GetObjectRequest.h>
+#include <aws/s3/model/HeadObjectRequest.h>
+#include <aws/s3/model/ListObjectsV2Request.h>
+#include <aws/core/client/AWSErrorMarshaller.h>
+#include <aws/core/endpoint/EndpointParameter.h>
+
+#include <IO/S3Common.h>
+#include <IO/S3/Requests.h>
+
+#include <Common/assert_cast.h>
+
+#include <Common/logger_useful.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
+namespace S3
+{
+
+Client::RetryStrategy::RetryStrategy(std::shared_ptr<Aws::Client::RetryStrategy> wrapped_strategy_)
+    : wrapped_strategy(std::move(wrapped_strategy_))
+{
+    if (!wrapped_strategy)
+        wrapped_strategy = Aws::Client::InitRetryStrategy();
+}
+
+/// NOLINTNEXTLINE(google-runtime-int)
+bool Client::RetryStrategy::ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const
+{
+    if (error.GetResponseCode() == Aws::Http::HttpResponseCode::MOVED_PERMANENTLY)
+        return false;
+
+    return wrapped_strategy->ShouldRetry(error, attemptedRetries);
+}
+
+/// NOLINTNEXTLINE(google-runtime-int)
+long Client::RetryStrategy::CalculateDelayBeforeNextRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const
+{
+    return wrapped_strategy->CalculateDelayBeforeNextRetry(error, attemptedRetries);
+}
+
+/// NOLINTNEXTLINE(google-runtime-int)
+long Client::RetryStrategy::GetMaxAttempts() const
+{
+    return wrapped_strategy->GetMaxAttempts();
+}
+
+void Client::RetryStrategy::GetSendToken()
+{
+    return wrapped_strategy->GetSendToken();
+}
+
+bool Client::RetryStrategy::HasSendToken()
+{
+    return wrapped_strategy->HasSendToken();
+}
+
+void Client::RetryStrategy::RequestBookkeeping(const Aws::Client::HttpResponseOutcome& httpResponseOutcome)
+{
+    return wrapped_strategy->RequestBookkeeping(httpResponseOutcome);
+}
+
+void Client::RetryStrategy::RequestBookkeeping(const Aws::Client::HttpResponseOutcome& httpResponseOutcome, const Aws::Client::AWSError<Aws::Client::CoreErrors>& lastError)
+{
+    return wrapped_strategy->RequestBookkeeping(httpResponseOutcome, lastError);
+}
+
+bool Client::checkIfWrongRegionDefined(const std::string & bucket, const Aws::S3::S3Error & error, std::string & region) const
+{
+    if (detect_region)
+        return false;
+
+    if (error.GetResponseCode() == Aws::Http::HttpResponseCode::BAD_REQUEST && error.GetExceptionName() == "AuthorizationHeaderMalformed")
+    {
+        region = GetErrorMarshaller()->ExtractRegion(error);
+
+        if (region.empty())
+            region = getRegionForBucket(bucket, /*force_detect*/ true);
+
+        assert(!explicit_region.empty());
+        if (region == explicit_region)
+            return false;
+
+        insertRegionOverride(bucket, region);
+        return true;
+    }
+
+    return false;
+}
+
+void Client::insertRegionOverride(const std::string & bucket, const std::string & region) const
+{
+    std::lock_guard lock(cache->region_cache_mutex);
+    auto [it, inserted] = cache->region_for_bucket_cache.emplace(bucket, region);
+    if (inserted)
+        LOG_INFO(log, "Detected different region ('{}') for bucket {} than the one defined ('{}')", region, bucket, explicit_region);
+}
+
+Model::HeadObjectOutcome Client::HeadObject(const HeadObjectRequest & request) const
+{
+    const auto & bucket = request.GetBucket();
+
+    if (auto region = getRegionForBucket(bucket); !region.empty())
+    {
+        if (!detect_region)
+            LOG_INFO(log, "Using region override {} for bucket {}", region, bucket);
+        request.overrideRegion(std::move(region));
+    }
+
+    if (auto uri = getURIForBucket(bucket); uri.has_value())
+        request.overrideURI(std::move(*uri));
+
+    auto result = Aws::S3::S3Client::HeadObject(request);
+    if (result.IsSuccess())
+        return result;
+
+    const auto & error = result.GetError();
+
+    std::string new_region;
+    if (checkIfWrongRegionDefined(bucket, error, new_region))
+    {
+        request.overrideRegion(new_region);
+        return HeadObject(request);
+    }
+
+    if (error.GetResponseCode() != Aws::Http::HttpResponseCode::MOVED_PERMANENTLY)
+        return result;
+
+    // maybe we detect a correct region
+    if (!detect_region)
+    {
+        if (auto region = GetErrorMarshaller()->ExtractRegion(error); !region.empty() && region != explicit_region)
+        {
+            request.overrideRegion(region);
+            insertRegionOverride(bucket, region);
+        }
+    }
+
+    auto bucket_uri = getURIForBucket(bucket);
+    if (!bucket_uri)
+    {
+        if (auto maybe_error = updateURIForBucketForHead(bucket); maybe_error.has_value())
+            return *maybe_error;
+
+        if (auto region = getRegionForBucket(bucket); !region.empty())
+        {
+            if (!detect_region)
+                LOG_INFO(log, "Using region override {} for bucket {}", region, bucket);
+            request.overrideRegion(std::move(region));
+        }
+
+        bucket_uri = getURIForBucket(bucket);
+        if (!bucket_uri)
+        {
+            LOG_ERROR(log, "Missing resolved URI for bucket {}, maybe the cache was cleaned", bucket);
+            return result;
+        }
+    }
+
+    const auto & current_uri_override = request.getURIOverride();
+    /// we already tried with this URI
+    if (current_uri_override && current_uri_override->uri == bucket_uri->uri)
+    {
+        LOG_INFO(log, "Getting redirected to the same invalid location {}", bucket_uri->uri.toString());
+        return result;
+    }
+
+    request.overrideURI(std::move(*bucket_uri));
+
+    return Aws::S3::S3Client::HeadObject(request);
+}
+
+Model::ListObjectsV2Outcome Client::ListObjectsV2(const ListObjectsV2Request & request) const
+{
+    return doRequest(request, [this](const Model::ListObjectsV2Request & req) { return Aws::S3::S3Client::ListObjectsV2(req); });
+}
+
+Model::ListObjectsOutcome Client::ListObjects(const ListObjectsRequest & request) const
+{
+    return doRequest(request, [this](const Model::ListObjectsRequest & req) { return Aws::S3::S3Client::ListObjects(req); });
+}
+
+Model::GetObjectOutcome Client::GetObject(const GetObjectRequest & request) const
+{
+    return doRequest(request, [this](const Model::GetObjectRequest & req) { return Aws::S3::S3Client::GetObject(req); });
+}
+
+Model::AbortMultipartUploadOutcome Client::AbortMultipartUpload(const AbortMultipartUploadRequest & request) const
+{
+    return doRequest(
+        request, [this](const Model::AbortMultipartUploadRequest & req) { return Aws::S3::S3Client::AbortMultipartUpload(req); });
+}
+
+Model::CreateMultipartUploadOutcome Client::CreateMultipartUpload(const CreateMultipartUploadRequest & request) const
+{
+    return doRequest(
+        request, [this](const Model::CreateMultipartUploadRequest & req) { return Aws::S3::S3Client::CreateMultipartUpload(req); });
+}
+
+Model::CompleteMultipartUploadOutcome Client::CompleteMultipartUpload(const CompleteMultipartUploadRequest & request) const
+{
+    return doRequest(
+        request, [this](const Model::CompleteMultipartUploadRequest & req) { return Aws::S3::S3Client::CompleteMultipartUpload(req); });
+}
+
+Model::CopyObjectOutcome Client::CopyObject(const CopyObjectRequest & request) const
+{
+    return doRequest(request, [this](const Model::CopyObjectRequest & req) { return Aws::S3::S3Client::CopyObject(req); });
+}
+
+Model::PutObjectOutcome Client::PutObject(const PutObjectRequest & request) const
+{
+    return doRequest(request, [this](const Model::PutObjectRequest & req) { return Aws::S3::S3Client::PutObject(req); });
+}
+
+Model::UploadPartOutcome Client::UploadPart(const UploadPartRequest & request) const
+{
+    return doRequest(request, [this](const Model::UploadPartRequest & req) { return Aws::S3::S3Client::UploadPart(req); });
+}
+
+Model::UploadPartCopyOutcome Client::UploadPartCopy(const UploadPartCopyRequest & request) const
+{
+    return doRequest(request, [this](const Model::UploadPartCopyRequest & req) { return Aws::S3::S3Client::UploadPartCopy(req); });
+}
+
+Model::DeleteObjectOutcome Client::DeleteObject(const DeleteObjectRequest & request) const
+{
+    return doRequest(request, [this](const Model::DeleteObjectRequest & req) { return Aws::S3::S3Client::DeleteObject(req); });
+}
+
+Model::DeleteObjectsOutcome Client::DeleteObjects(const DeleteObjectsRequest & request) const
+{
+    return doRequest(request, [this](const Model::DeleteObjectsRequest & req) { return Aws::S3::S3Client::DeleteObjects(req); });
+}
+
+std::string Client::getRegionForBucket(const std::string & bucket, bool force_detect) const
+{
+    std::lock_guard lock(cache->region_cache_mutex);
+    if (auto it = cache->region_for_bucket_cache.find(bucket); it != cache->region_for_bucket_cache.end())
+        return it->second;
+
+    if (!force_detect && !detect_region)
+        return "";
+
+
+    LOG_INFO(log, "Resolving region for bucket {}", bucket);
+    Aws::S3::Model::HeadBucketRequest req;
+    req.SetBucket(bucket);
+
+    std::string region;
+    auto outcome = HeadBucket(req);
+    if (outcome.IsSuccess())
+    {
+        const auto & result = outcome.GetResult();
+        region = result.GetRegion();
+    }
+    else
+    {
+        static const std::string region_header = "x-amz-bucket-region";
+        const auto & headers = outcome.GetError().GetResponseHeaders();
+        if (auto it = headers.find(region_header); it != headers.end())
+            region = it->second;
+    }
+
+    if (region.empty())
+    {
+        LOG_INFO(log, "Failed resolving region for bucket {}", bucket);
+        return "";
+    }
+
+    LOG_INFO(log, "Found region {} for bucket {}", region, bucket);
+
+    auto [it, _] = cache->region_for_bucket_cache.emplace(bucket, std::move(region));
+
+    return it->second;
+}
+
+std::optional<S3::URI> Client::getURIFromError(const Aws::S3::S3Error & error) const
+{
+    auto endpoint = GetErrorMarshaller()->ExtractEndpoint(error);
+    if (endpoint.empty())
+        return std::nullopt;
+
+    auto & s3_client = const_cast<Client &>(*this);
+    const auto * endpoint_provider = dynamic_cast<Aws::S3::Endpoint::S3DefaultEpProviderBase *>(s3_client.accessEndpointProvider().get());
+    auto resolved_endpoint = endpoint_provider->ResolveEndpoint({});
+
+    if (!resolved_endpoint.IsSuccess())
+        return std::nullopt;
+
+    auto uri = resolved_endpoint.GetResult().GetURI();
+    uri.SetAuthority(endpoint);
+
+    return S3::URI(uri.GetURIString());
+}
+
+// Do a list request because head requests don't have body in response
+std::optional<Aws::S3::S3Error> Client::updateURIForBucketForHead(const std::string & bucket) const
+{
+    ListObjectsV2Request req;
+    req.SetBucket(bucket);
+    req.SetMaxKeys(1);
+    auto result = ListObjectsV2(req);
+    if (result.IsSuccess())
+        return std::nullopt;
+    return result.GetError();
+}
+
+std::optional<S3::URI> Client::getURIForBucket(const std::string & bucket) const
+{
+    std::lock_guard lock(cache->uri_cache_mutex);
+    if (auto it = cache->uri_for_bucket_cache.find(bucket); it != cache->uri_for_bucket_cache.end())
+        return it->second;
+
+    return std::nullopt;
+}
+
+void Client::updateURIForBucket(const std::string & bucket, S3::URI new_uri) const
+{
+    std::lock_guard lock(cache->uri_cache_mutex);
+    if (auto it = cache->uri_for_bucket_cache.find(bucket); it != cache->uri_for_bucket_cache.end())
+    {
+        if (it->second.uri == new_uri.uri)
+            return;
+
+        LOG_INFO(log, "Updating URI for bucket {} to {}", bucket, new_uri.uri.toString());
+        it->second = std::move(new_uri);
+
+        return;
+    }
+
+    LOG_INFO(log, "Updating URI for bucket {} to {}", bucket, new_uri.uri.toString());
+    cache->uri_for_bucket_cache.emplace(bucket, std::move(new_uri));
+}
+
+
+void ClientCache::clearCache()
+{
+    {
+        std::lock_guard lock(region_cache_mutex);
+        region_for_bucket_cache.clear();
+    }
+    {
+        std::lock_guard lock(uri_cache_mutex);
+        uri_for_bucket_cache.clear();
+    }
+}
+
+void ClientCacheRegistry::registerClient(const std::shared_ptr<ClientCache> & client_cache)
+{
+    std::lock_guard lock(clients_mutex);
+    auto [it, inserted] = client_caches.emplace(client_cache.get(), client_cache);
+    if (!inserted)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Same S3 client registered twice");
+}
+
+void ClientCacheRegistry::unregisterClient(ClientCache * client)
+{
+    std::lock_guard lock(clients_mutex);
+    auto erased = client_caches.erase(client);
+    if (erased == 0)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't unregister S3 client, either it was already unregistered or not registered at all");
+}
+
+void ClientCacheRegistry::clearCacheForAll()
+{
+    std::lock_guard lock(clients_mutex);
+
+    for (auto it = client_caches.begin(); it != client_caches.end();)
+    {
+        if (auto locked_client = it->second.lock(); locked_client)
+        {
+            locked_client->clearCache();
+            ++it;
+        }
+        else
+        {
+            LOG_INFO(&Poco::Logger::get("ClientCacheRegistry"), "Deleting leftover S3 client cache");
+            it = client_caches.erase(it);
+        }
+    }
+
+}
+
+}
+
+}
+
+#endif