Merge remote-tracking branch 'upstream/master' into ncb/hostname-system-log-tables

.github/actions/common_setup/action.yml

@@ -19,6 +19,8 @@ runs:
           cat >> "$GITHUB_ENV" << 'EOF'
           TEMP_PATH=${{runner.temp}}/${{inputs.job_type}}
           REPO_COPY=${{runner.temp}}/${{inputs.job_type}}/git-repo-copy
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPORTS_PATH=${{runner.temp}}/reports_dir
           EOF
           if [ -z "${{env.GITHUB_JOB_OVERRIDDEN}}" ] && [ "true" == "${{inputs.nested_job}}" ]; then
             echo "The GITHUB_JOB_OVERRIDDEN ENV is unset, and must be set for the nested jobs"

.github/workflows/backport_branches.yml

@@ -105,66 +105,22 @@ jobs:
           path: ${{ runner.temp }}/changed_images.json
   CompatibilityCheckX86:
     needs: [BuilderDebRelease]
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/compatibility_check
-          REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          EOF
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: CompatibilityCheckX86
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Compatibility check X86
+      runner_type: style-checker
+      run_command: |
+        cd "$REPO_COPY/tests/ci"
+        python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions
   CompatibilityCheckAarch64:
     needs: [BuilderDebAarch64]
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/compatibility_check
-          REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          EOF
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: CompatibilityCheckAarch64
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Compatibility check X86
+      runner_type: style-checker
+      run_command: |
+        cd "$REPO_COPY/tests/ci"
+        python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
 #########################################################################################
 #################################### ORDINARY BUILDS ####################################
 #########################################################################################
@@ -239,303 +195,114 @@ jobs:
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
   BuilderReport:
+    if: ${{ success() || failure() }}
     needs:
       - BuilderDebRelease
       - BuilderDebAarch64
       - BuilderDebAsan
       - BuilderDebTsan
       - BuilderDebDebug
-    runs-on: [self-hosted, style-checker]
-    if: ${{ success() || failure() }}
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          CHECK_NAME=ClickHouse build check
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          TEMP_PATH=${{runner.temp}}/report_check
-          NEEDS_DATA_PATH=${{runner.temp}}/needs.json
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Report Builder
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cat > "$NEEDS_DATA_PATH" << 'EOF'
-          ${{ toJSON(needs) }}
-          EOF
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 build_report_check.py "$CHECK_NAME"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: ClickHouse build check
+      runner_type: style-checker
+      additional_envs: |
+        NEEDS_DATA<<NDENV
+        ${{ toJSON(needs) }}
+        NDENV
+      run_command: |
+        cd "$GITHUB_WORKSPACE/tests/ci"
+        python3 build_report_check.py "$CHECK_NAME"
   BuilderSpecialReport:
+    if: ${{ success() || failure() }}
     needs:
       - BuilderBinDarwin
       - BuilderBinDarwinAarch64
-    runs-on: [self-hosted, style-checker]
-    if: ${{ success() || failure() }}
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/report_check
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=ClickHouse special build check
-          NEEDS_DATA_PATH=${{runner.temp}}/needs.json
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Report Builder
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cat > "$NEEDS_DATA_PATH" << 'EOF'
-          ${{ toJSON(needs) }}
-          EOF
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 build_report_check.py "$CHECK_NAME"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: ClickHouse special build check
+      runner_type: style-checker
+      additional_envs: |
+        NEEDS_DATA<<NDENV
+        ${{ toJSON(needs) }}
+        NDENV
+      run_command: |
+        cd "$GITHUB_WORKSPACE/tests/ci"
+        python3 build_report_check.py "$CHECK_NAME"
 ############################################################################################
 #################################### INSTALL PACKAGES ######################################
 ############################################################################################
   InstallPackagesTestRelease:
     needs: [BuilderDebRelease]
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/test_install
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=Install packages (amd64)
-          REPO_COPY=${{runner.temp}}/test_install/ClickHouse
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Test packages installation
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 install_check.py "$CHECK_NAME"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Install packages (amd64)
+      runner_type: style-checker
+      run_command: |
+        cd "$REPO_COPY/tests/ci"
+        python3 install_check.py "$CHECK_NAME"
   InstallPackagesTestAarch64:
     needs: [BuilderDebAarch64]
-    runs-on: [self-hosted, style-checker-aarch64]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/test_install
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=Install packages (arm64)
-          REPO_COPY=${{runner.temp}}/test_install/ClickHouse
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Test packages installation
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 install_check.py "$CHECK_NAME"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Install packages (arm64)
+      runner_type: style-checker-aarch64
+      run_command: |
+        cd "$REPO_COPY/tests/ci"
+        python3 install_check.py "$CHECK_NAME"
 ##############################################################################################
 ########################### FUNCTIONAl STATELESS TESTS #######################################
 ##############################################################################################
   FunctionalStatelessTestAsan:
     needs: [BuilderDebAsan]
-    runs-on: [self-hosted, func-tester]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/stateless_debug
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=Stateless tests (asan)
-          REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
-          KILL_TIMEOUT=10800
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Functional test
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Stateless tests (asan)
+      runner_type: func-tester
+      additional_envs: |
+        KILL_TIMEOUT=10800
+      run_command: |
+        cd "$REPO_COPY/tests/ci"
+        python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
 ##############################################################################################
 ############################ FUNCTIONAl STATEFUL TESTS #######################################
 ##############################################################################################
   FunctionalStatefulTestDebug:
     needs: [BuilderDebDebug]
-    runs-on: [self-hosted, func-tester]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/stateful_debug
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=Stateful tests (debug)
-          REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
-          KILL_TIMEOUT=3600
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Functional test
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Stateful tests (debug)
+      runner_type: func-tester
+      additional_envs: |
+        KILL_TIMEOUT=3600
+      run_command: |
+        cd "$REPO_COPY/tests/ci"
+        python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
 ##############################################################################################
 ######################################### STRESS TESTS #######################################
 ##############################################################################################
   StressTestTsan:
     needs: [BuilderDebTsan]
-    # func testers have 16 cores + 128 GB memory
-    # while stress testers have 36 cores + 72 memory
-    # It would be better to have something like 32 + 128,
-    # but such servers almost unavailable as spot instances.
-    runs-on: [self-hosted, func-tester]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/stress_thread
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=Stress test (tsan)
-          REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Stress test
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 stress_check.py "$CHECK_NAME"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Stress test (tsan)
+      runner_type: stress-tester
+      run_command: |
+        cd "$REPO_COPY/tests/ci"
+        python3 stress_check.py "$CHECK_NAME"
 #############################################################################################
 ############################# INTEGRATION TESTS #############################################
 #############################################################################################
   IntegrationTestsRelease:
     needs: [BuilderDebRelease]
-    runs-on: [self-hosted, stress-tester]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/integration_tests_release
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=Integration tests (release)
-          REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
-          EOF
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Integration test
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 integration_test_check.py "$CHECK_NAME"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Integration tests (release)
+      runner_type: stress-tester
+      batches: 4
+      run_command: |
+        cd "$REPO_COPY/tests/ci"
+        python3 integration_test_check.py "$CHECK_NAME"
   FinishCheck:
     needs:
       - DockerHubPush

.github/workflows/docs_check.yml

@@ -96,68 +96,30 @@ jobs:
           path: ${{ runner.temp }}/changed_images.json
   StyleCheck:
     needs: DockerHubPush
-    runs-on: [self-hosted, style-checker]
-    if: ${{ success() || failure() }}
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{ runner.temp }}/style_check
-          ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
-          ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
-          RCSK
-          EOF
-      - name: Download changed images
-        # even if artifact does not exist, e.g. on `do not test` label or failed Docker job
-        continue-on-error: true
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.TEMP_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Style Check
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
+    # We need additional `&& ! cancelled()` to have the job being able to cancel
+    if: ${{ success() || failure() || ( always() && ! cancelled() ) }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Style check
+      runner_type: style-checker
+      run_command: |
+          cd "$REPO_COPY/tests/ci"
           python3 style_check.py
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    secrets:
+      secret_envs: |
+        ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
+        ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
+        RCSK
   DocsCheck:
     needs: DockerHubPush
-    runs-on: [self-hosted, func-tester-aarch64]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/docs_check
-          REPO_COPY=${{runner.temp}}/docs_check/ClickHouse
-          EOF
-      - name: Download changed images
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.TEMP_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Docs Check
-        run: |
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 docs_check.py
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Docs check
+      runner_type: func-tester-aarch64
+      additional_envs: |
+      run_command: |
+        cd "$REPO_COPY/tests/ci"
+        python3 docs_check.py
   FinishCheck:
     needs:
       - StyleCheck

.github/workflows/jepsen.yml

@@ -11,60 +11,19 @@ on: # yamllint disable-line rule:truthy
   workflow_call:
 jobs:
   KeeperJepsenRelease:
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/keeper_jepsen
-          REPO_COPY=${{runner.temp}}/keeper_jepsen/ClickHouse
-          EOF
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-          fetch-depth: 0
-          filter: tree:0
-      - name: Jepsen Test
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 jepsen_check.py keeper
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Jepsen keeper check
+      runner_type: style-checker
+      run_command: |
+        cd "$REPO_COPY/tests/ci"
+        python3 jepsen_check.py keeper
   # ServerJepsenRelease:
   #   runs-on: [self-hosted, style-checker]
-  #   if: ${{ always() }}
-  #   needs: [KeeperJepsenRelease]
-  #   steps:
-  #     - name: Set envs
-  #       run: |
-  #         cat >> "$GITHUB_ENV" << 'EOF'
-  #         TEMP_PATH=${{runner.temp}}/server_jepsen
-  #         REPO_COPY=${{runner.temp}}/server_jepsen/ClickHouse
-  #         EOF
-  #     - name: Check out repository code
-  #       uses: ClickHouse/checkout@v1
-  #       with:
-  #         clear-repository: true
-  #         fetch-depth: 0
-  #         filter: tree:0
-  #     - name: Jepsen Test
-  #       run: |
-  #         sudo rm -fr "$TEMP_PATH"
-  #         mkdir -p "$TEMP_PATH"
-  #         cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-  #         cd "$REPO_COPY/tests/ci"
-  #         python3 jepsen_check.py server
-  #     - name: Cleanup
-  #       if: always()
-  #       run: |
-  #         docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-  #         docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-  #         sudo rm -fr "$TEMP_PATH"
+  #   uses: ./.github/workflows/reusable_test.yml
+  #   with:
+  #     test_name: Jepsen server check
+  #     runner_type: style-checker
+  #     run_command: |
+  #       cd "$REPO_COPY/tests/ci"
+  #       python3 jepsen_check.py server

.github/workflows/libfuzzer.yml

@@ -10,86 +10,17 @@ on: # yamllint disable-line rule:truthy
   workflow_call:
 jobs:
   BuilderFuzzers:
-    runs-on: [self-hosted, builder]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/build_check
-          IMAGES_PATH=${{runner.temp}}/images_path
-          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
-          CACHES_PATH=${{runner.temp}}/../ccaches
-          BUILD_NAME=fuzzers
-          EOF
-      - name: Download changed images
-        # even if artifact does not exist, e.g. on `do not test` label or failed Docker job
-        continue-on-error: true
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.IMAGES_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-          submodules: true
-          ref: ${{github.ref}}
-      - name: Build
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
-      - name: Upload build URLs to artifacts
-        if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.BUILD_URLS }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+    uses: ./.github/workflows/reusable_build.yml
+    with:
+      build_name: fuzzers
   libFuzzerTest:
     needs: [BuilderFuzzers]
-    runs-on: [self-hosted, func-tester]
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/libfuzzer
-          REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME=libFuzzer tests
-          REPO_COPY=${{runner.temp}}/libfuzzer/ClickHouse
-          KILL_TIMEOUT=10800
-          EOF
-      - name: Download changed images
-        # even if artifact does not exist, e.g. on `do not test` label or failed Docker job
-        continue-on-error: true
-        uses: actions/download-artifact@v3
-        with:
-          name: changed_images
-          path: ${{ env.TEMP_PATH }}
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.REPORTS_PATH }}
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: libFuzzer test
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci"
-          python3 libfuzzer_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: libFuzzer tests
+      runner_type: func-tester
+      additional_envs: |
+        KILL_TIMEOUT=10800
+      run_command: |
+        cd "$REPO_COPY/tests/ci"
+        python3 libfuzzer_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"

.github/workflows/nightly.yml

@@ -74,9 +74,6 @@ jobs:
         with:
           name: changed_images
           path: ${{ runner.temp }}/changed_images.json
-  Codebrowser:
-    needs: [DockerHubPush]
-    uses: ./.github/workflows/woboq.yml
   SonarCloud:
     runs-on: [self-hosted, builder]
     env:

.github/workflows/reusable_build.yml

@@ -1,6 +1,10 @@
 ### For the pure soul wishes to move it to another place
 # https://github.com/orgs/community/discussions/9050
 
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
 name: Build ClickHouse
 'on':
   workflow_call:
@@ -25,6 +29,8 @@ name: Build ClickHouse
 jobs:
   Build:
     name: Build-${{inputs.build_name}}
+    env:
+      GITHUB_JOB_OVERRIDDEN: Build-${{inputs.build_name}}
     runs-on: [self-hosted, '${{inputs.runner_type}}']
     steps:
       - name: Check out repository code
@@ -37,8 +43,6 @@ jobs:
       - name: Set build envs
         run: |
           cat >> "$GITHUB_ENV" << 'EOF'
-          IMAGES_PATH=${{runner.temp}}/images_path
-          GITHUB_JOB_OVERRIDDEN=Build-${{inputs.build_name}}
           ${{inputs.additional_envs}}
           EOF
           python3 "$GITHUB_WORKSPACE"/tests/ci/ci_config.py --build-name "${{inputs.build_name}}" >> "$GITHUB_ENV"
@@ -71,4 +75,5 @@ jobs:
           name: ${{ env.BUILD_URLS }}
           path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Clean
+        if: always()
         uses: ./.github/actions/clean

.github/workflows/reusable_test.yml

@@ -0,0 +1,113 @@
+### For the pure soul wishes to move it to another place
+# https://github.com/orgs/community/discussions/9050
+
+name: Testing workflow
+'on':
+  workflow_call:
+    inputs:
+      test_name:
+        description: the value of test type from tests/ci/ci_config.py, ends up as $CHECK_NAME ENV
+        required: true
+        type: string
+      runner_type:
+        description: the label of runner to use
+        required: true
+        type: string
+      run_command:
+        description: the command to launch the check. Usually starts with `cd '$REPO_COPY/tests/ci'`
+        required: true
+        type: string
+      batches:
+        description: how many batches for the test will be launched
+        default: 1
+        type: number
+      checkout_depth:
+        description: the value of the git shallow checkout
+        required: false
+        type: number
+        default: 1
+      submodules:
+        description: if the submodules should be checked out
+        required: false
+        type: boolean
+        default: false
+      additional_envs:
+        description: additional ENV variables to setup the job
+        type: string
+    secrets:
+      secret_envs:
+        description: if given, it's passed to the environments
+        required: false
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+  CHECK_NAME: ${{inputs.test_name}}
+
+jobs:
+  PrepareStrategy:
+    # batches < 1 is misconfiguration,
+    # and we need this step only for batches > 1
+    if: ${{ inputs.batches > 1 }}
+    runs-on: [self-hosted, style-checker-aarch64]
+    outputs:
+      batches: ${{steps.batches.outputs.batches}}
+    steps:
+      - name: Calculate batches
+        id: batches
+        run: |
+          batches_output=$(python3 -c 'import json; print(json.dumps(list(range(${{inputs.batches}}))))')
+          echo "batches=${batches_output}" >> "$GITHUB_OUTPUT"
+  Test:
+    # If PrepareStrategy is skipped for batches == 1,
+    # we still need to launch the test.
+    # `! failure()` is mandatory here to launch on skipped Job
+    # `&& !cancelled()` to allow the be cancelable
+    if: ${{ ( !failure() && !cancelled() ) && inputs.batches > 0 }}
+    # Do not add `-0` to the end, if there's only one batch
+    name: ${{inputs.test_name}}${{ inputs.batches > 1 && format('-{0}',matrix.batch) || '' }}
+    env:
+      GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}${{ inputs.batches > 1 && format('-{0}',matrix.batch) || '' }}
+    runs-on: [self-hosted, '${{inputs.runner_type}}']
+    needs: [PrepareStrategy]
+    strategy:
+      fail-fast: false  # we always wait for entire matrix
+      matrix:
+        # if PrepareStrategy does not have batches, we use 0
+        batch: ${{ needs.PrepareStrategy.outputs.batches
+          && fromJson(needs.PrepareStrategy.outputs.batches)
+          || fromJson('[0]')}}
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          submodules: ${{inputs.submodules}}
+          fetch-depth: ${{inputs.checkout_depth}}
+          filter: tree:0
+      - name: Set build envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          ${{inputs.additional_envs}}
+          ${{secrets.secret_envs}}
+          EOF
+      - name: Common setup
+        uses: ./.github/actions/common_setup
+        with:
+          job_type: test
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Setup batch
+        if: ${{ inputs.batches > 1}}
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          RUN_BY_HASH_NUM=${{matrix.batch}}
+          RUN_BY_HASH_TOTAL=${{inputs.batches}}
+          EOF
+      - name: Run test
+        run: ${{inputs.run_command}}
+      - name: Clean
+        if: always()
+        uses: ./.github/actions/clean

.github/workflows/woboq.yml

@@ -1,44 +0,0 @@
-name: WoboqBuilder
-env:
-  # Force the stdout and stderr streams to be unbuffered
-  PYTHONUNBUFFERED: 1
-
-concurrency:
-  group: woboq
-on: # yamllint disable-line rule:truthy
-  workflow_dispatch:
-  workflow_call:
-jobs:
-  # don't use dockerhub push because this image updates so rarely
-  WoboqCodebrowser:
-    runs-on: [self-hosted, style-checker]
-    timeout-minutes: 420  # the task is pretty heavy, so there's an additional hour
-    steps:
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/codebrowser
-          REPO_COPY=${{runner.temp}}/codebrowser/ClickHouse
-          IMAGES_PATH=${{runner.temp}}/images_path
-          EOF
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-          submodules: 'true'
-      - name: Download json reports
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ env.IMAGES_PATH }}
-      - name: Codebrowser
-        run: |
-          sudo rm -fr "$TEMP_PATH"
-          mkdir -p "$TEMP_PATH"
-          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 codebrowser_check.py
-      - name: Cleanup
-        if: always()
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"

CMakeLists.txt

@@ -164,7 +164,7 @@ if (OS_LINUX)
     #   and whatever is poisoning it by LD_PRELOAD should not link to our symbols.
     # - The clickhouse-odbc-bridge and clickhouse-library-bridge binaries
     #   should not expose their symbols to ODBC drivers and libraries.
-    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-export-dynamic")
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-export-dynamic -Wl,--gc-sections")
 endif ()
 
 if (OS_DARWIN)
@@ -273,6 +273,11 @@ option (ENABLE_BUILD_PROFILING "Enable profiling of build time" OFF)
 if (ENABLE_BUILD_PROFILING)
      if (COMPILER_CLANG)
         set (COMPILER_FLAGS "${COMPILER_FLAGS} -ftime-trace")
+
+        if (LINKER_NAME MATCHES "lld")
+            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--time-trace")
+            set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,--time-trace")
+        endif ()
      else ()
         message (${RECONFIGURE_MESSAGE_LEVEL} "Build profiling is only available with CLang")
      endif ()
@@ -315,7 +320,7 @@ endif ()
 set (COMPILER_FLAGS "${COMPILER_FLAGS}")
 
 # Our built-in unwinder only supports DWARF version up to 4.
-set (DEBUG_INFO_FLAGS "-g -gdwarf-4")
+set (DEBUG_INFO_FLAGS "-g")
 
 # Disable omit frame pointer compiler optimization using -fno-omit-frame-pointer
 option(DISABLE_OMIT_FRAME_POINTER "Disable omit frame pointer compiler optimization" OFF)
@@ -554,6 +559,13 @@ if (ENABLE_RUST)
     endif()
 endif()
 
+if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND NOT SANITIZE AND OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64))
+    set(CHECK_LARGE_OBJECT_SIZES_DEFAULT ON)
+else ()
+    set(CHECK_LARGE_OBJECT_SIZES_DEFAULT OFF)
+endif ()
+option(CHECK_LARGE_OBJECT_SIZES "Check that there are no large object files after build." ${CHECK_LARGE_OBJECT_SIZES_DEFAULT})
+
 add_subdirectory (base)
 add_subdirectory (src)
 add_subdirectory (programs)

base/base/wide_integer_impl.h

@@ -65,7 +65,7 @@ class IsTupleLike
     static void check(...);
 
 public:
-    static constexpr const bool value = !std::is_void<decltype(check<T>(nullptr))>::value;
+    static constexpr const bool value = !std::is_void_v<decltype(check<T>(nullptr))>;
 };
 
 }
@@ -79,7 +79,7 @@ class numeric_limits<wide::integer<Bits, Signed>>
 {
 public:
     static constexpr bool is_specialized = true;
-    static constexpr bool is_signed = is_same<Signed, signed>::value;
+    static constexpr bool is_signed = is_same_v<Signed, signed>;
     static constexpr bool is_integer = true;
     static constexpr bool is_exact = true;
     static constexpr bool has_infinity = false;
@@ -91,7 +91,7 @@ public:
     static constexpr bool is_iec559 = false;
     static constexpr bool is_bounded = true;
     static constexpr bool is_modulo = true;
-    static constexpr int digits = Bits - (is_same<Signed, signed>::value ? 1 : 0);
+    static constexpr int digits = Bits - (is_same_v<Signed, signed> ? 1 : 0);
     static constexpr int digits10 = digits * 0.30103 /*std::log10(2)*/;
     static constexpr int max_digits10 = 0;
     static constexpr int radix = 2;
@@ -104,7 +104,7 @@ public:
 
     static constexpr wide::integer<Bits, Signed> min() noexcept
     {
-        if (is_same<Signed, signed>::value)
+        if constexpr (is_same_v<Signed, signed>)
         {
             using T = wide::integer<Bits, signed>;
             T res{};
@@ -118,7 +118,7 @@ public:
     {
         using T = wide::integer<Bits, Signed>;
         T res{};
-        res.items[T::_impl::big(0)] = is_same<Signed, signed>::value
+        res.items[T::_impl::big(0)] = is_same_v<Signed, signed>
             ? std::numeric_limits<typename wide::integer<Bits, Signed>::signed_base_type>::max()
             : std::numeric_limits<typename wide::integer<Bits, Signed>::base_type>::max();
         for (unsigned i = 1; i < wide::integer<Bits, Signed>::_impl::item_count; ++i)

base/glibc-compatibility/CMakeLists.txt

@@ -5,9 +5,6 @@ if (GLIBC_COMPATIBILITY)
     endif()
 
     enable_language(ASM)
-    include(CheckIncludeFile)
-
-    check_include_file("sys/random.h" HAVE_SYS_RANDOM_H)
 
     add_headers_and_sources(glibc_compatibility .)
     add_headers_and_sources(glibc_compatibility musl)
@@ -21,11 +18,6 @@ if (GLIBC_COMPATIBILITY)
         message (FATAL_ERROR "glibc_compatibility can only be used on x86_64 or aarch64.")
     endif ()
 
-    list(REMOVE_ITEM glibc_compatibility_sources musl/getentropy.c)
-    if(HAVE_SYS_RANDOM_H)
-        list(APPEND glibc_compatibility_sources musl/getentropy.c)
-    endif()
-
     # Need to omit frame pointers to match the performance of glibc
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fomit-frame-pointer")
 

cmake/cpu_features.cmake

@@ -1,10 +1,5 @@
 # https://software.intel.com/sites/landingpage/IntrinsicsGuide/
 
-include (CheckCXXSourceCompiles)
-include (CMakePushCheckState)
-
-cmake_push_check_state ()
-
 # The variables HAVE_* determine if compiler has support for the flag to use the corresponding instruction set.
 # The options ENABLE_* determine if we will tell compiler to actually use the corresponding instruction set if compiler can do it.
 
@@ -137,178 +132,53 @@ elseif (ARCH_AMD64)
     endif()
 
     # ClickHouse can be cross-compiled (e.g. on an ARM host for x86) but it is also possible to build ClickHouse on x86 w/o AVX for x86 w/
-    # AVX. We only check that the compiler can emit certain SIMD instructions, we don't care if the host system is able to run the binary.
-    # Therefore, use check_cxx_source_compiles (= does the code compile+link?) instead of check_cxx_source_runs (= does the code
-    # compile+link+run).
+    # AVX. We only assume that the compiler can emit certain SIMD instructions, we don't care if the host system is able to run the binary.
+
+    SET (HAVE_SSSE3 1)
+    SET (HAVE_SSE41 1)
+    SET (HAVE_SSE42 1)
+    SET (HAVE_PCLMULQDQ 1)
+    SET (HAVE_POPCNT 1)
+    SET (HAVE_AVX 1)
+    SET (HAVE_AVX2 1)
+    SET (HAVE_AVX512 1)
+    SET (HAVE_AVX512_VBMI 1)
+    SET (HAVE_BMI 1)
+    SET (HAVE_BMI2 1)
 
-    set (TEST_FLAG "-mssse3")
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
-    check_cxx_source_compiles("
-        #include <tmmintrin.h>
-        int main() {
-            __m64 a = _mm_abs_pi8(__m64());
-            (void)a;
-            return 0;
-        }
-    " HAVE_SSSE3)
     if (HAVE_SSSE3 AND ENABLE_SSSE3)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -mssse3")
     endif ()
-
-    set (TEST_FLAG "-msse4.1")
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
-    check_cxx_source_compiles("
-        #include <smmintrin.h>
-        int main() {
-            auto a = _mm_insert_epi8(__m128i(), 0, 0);
-            (void)a;
-            return 0;
-        }
-    " HAVE_SSE41)
     if (HAVE_SSE41 AND ENABLE_SSE41)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -msse4.1")
     endif ()
-
-    set (TEST_FLAG "-msse4.2")
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
-    check_cxx_source_compiles("
-        #include <nmmintrin.h>
-        int main() {
-            auto a = _mm_crc32_u64(0, 0);
-            (void)a;
-            return 0;
-        }
-    " HAVE_SSE42)
     if (HAVE_SSE42 AND ENABLE_SSE42)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -msse4.2")
     endif ()
-
-    set (TEST_FLAG "-mpclmul")
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
-    check_cxx_source_compiles("
-        #include <wmmintrin.h>
-        int main() {
-            auto a = _mm_clmulepi64_si128(__m128i(), __m128i(), 0);
-            (void)a;
-            return 0;
-        }
-    " HAVE_PCLMULQDQ)
     if (HAVE_PCLMULQDQ AND ENABLE_PCLMULQDQ)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -mpclmul")
     endif ()
-
-    set (TEST_FLAG "-mpopcnt")
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
-    check_cxx_source_compiles("
-        int main() {
-            auto a = __builtin_popcountll(0);
-            (void)a;
-            return 0;
-        }
-    " HAVE_POPCNT)
     if (HAVE_POPCNT AND ENABLE_POPCNT)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -mpopcnt")
     endif ()
-
-    set (TEST_FLAG "-mavx")
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
-    check_cxx_source_compiles("
-        #include <immintrin.h>
-        int main() {
-            auto a = _mm256_insert_epi8(__m256i(), 0, 0);
-            (void)a;
-            return 0;
-        }
-    " HAVE_AVX)
     if (HAVE_AVX AND ENABLE_AVX)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx")
     endif ()
-
-    set (TEST_FLAG "-mavx2")
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
-    check_cxx_source_compiles("
-        #include <immintrin.h>
-        int main() {
-            auto a = _mm256_add_epi16(__m256i(), __m256i());
-            (void)a;
-            return 0;
-        }
-    " HAVE_AVX2)
     if (HAVE_AVX2 AND ENABLE_AVX2)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx2")
     endif ()
-
-    set (TEST_FLAG "-mavx512f -mavx512bw -mavx512vl")
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
-    check_cxx_source_compiles("
-        #include <immintrin.h>
-        int main() {
-            auto a = _mm512_setzero_epi32();
-            (void)a;
-            auto b = _mm512_add_epi16(__m512i(), __m512i());
-            (void)b;
-            auto c = _mm_cmp_epi8_mask(__m128i(), __m128i(), 0);
-            (void)c;
-            return 0;
-        }
-    " HAVE_AVX512)
     if (HAVE_AVX512 AND ENABLE_AVX512)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512f -mavx512bw -mavx512vl")
     endif ()
-
-    set (TEST_FLAG "-mavx512vbmi")
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
-    check_cxx_source_compiles("
-        #include <immintrin.h>
-        int main() {
-            auto a = _mm512_permutexvar_epi8(__m512i(), __m512i());
-            (void)a;
-            return 0;
-        }
-    " HAVE_AVX512_VBMI)
     if (HAVE_AVX512 AND ENABLE_AVX512 AND HAVE_AVX512_VBMI AND ENABLE_AVX512_VBMI)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512vbmi")
     endif ()
-
-    set (TEST_FLAG "-mbmi")
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
-    check_cxx_source_compiles("
-        #include <immintrin.h>
-        int main() {
-            auto a = _blsr_u32(0);
-            (void)a;
-            return 0;
-        }
-    " HAVE_BMI)
     if (HAVE_BMI AND ENABLE_BMI)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi")
     endif ()
-
-    set (TEST_FLAG "-mbmi2")
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
-    check_cxx_source_compiles("
-        #include <immintrin.h>
-        int main() {
-            auto a = _pdep_u64(0, 0);
-            (void)a;
-            return 0;
-        }
-    " HAVE_BMI2)
     if (HAVE_BMI2 AND HAVE_AVX2 AND ENABLE_AVX2 AND ENABLE_BMI2)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi2")
     endif ()
-
-    # Limit avx2/avx512 flag for specific source build
-    set (X86_INTRINSICS_FLAGS "")
-    if (ENABLE_AVX2_FOR_SPEC_OP)
-        if (HAVE_BMI)
-            set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mbmi")
-        endif ()
-        if (HAVE_AVX AND HAVE_AVX2)
-            set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx -mavx2")
-        endif ()
-    endif ()
-
     if (ENABLE_AVX512_FOR_SPEC_OP)
         set (X86_INTRINSICS_FLAGS "")
         if (HAVE_BMI)
@@ -321,5 +191,3 @@ elseif (ARCH_AMD64)
 else ()
     # RISC-V + exotic platforms
 endif ()
-
-cmake_pop_check_state ()

contrib/AMQP-CPP

@@ -1 +1 @@
-Subproject commit 818c2d8ad96a08a5d20fece7d1e1e8855a2b0860
+Subproject commit 00f09897ce020a84e38f87dc416af4a19c5da9ae

contrib/CMakeLists.txt

@@ -1,7 +1,7 @@
 #"${folder}/CMakeLists.txt" Third-party libraries may have substandard code.
 
-set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
+set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -ffunction-sections -fdata-sections")
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -ffunction-sections -fdata-sections")
 
 if (WITH_COVERAGE)
   set (WITHOUT_COVERAGE_LIST ${WITHOUT_COVERAGE})

contrib/aws-cmake/AwsFeatureTests.cmake

@@ -1,114 +1,13 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
 
-include(CheckCSourceRuns)
-
 option(USE_CPU_EXTENSIONS "Whenever possible, use functions optimized for CPUs with specific extensions (ex: SSE, AVX)." ON)
 
-# In the current (11/2/21) state of mingw64, the packaged gcc is not capable of emitting properly aligned avx2 instructions under certain circumstances.
-# This leads to crashes for windows builds using mingw64 when invoking the avx2-enabled versions of certain functions.  Until we can find a better
-# work-around, disable avx2 (and all other extensions) in mingw builds.
-#
-# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412
-#
-if (MINGW)
-    message(STATUS "MINGW detected!  Disabling avx2 and other CPU extensions")
-    set(USE_CPU_EXTENSIONS OFF)
-endif()
+if (ARCH_AMD64)
+    set (AWS_ARCH_INTEL 1)
+elseif (ARCH_AARCH64)
+    set (AWS_ARCH_ARM64 1)
+endif ()
 
-if(NOT CMAKE_CROSSCOMPILING)
-    check_c_source_runs("
-    #include <stdbool.h>
-    bool foo(int a, int b, int *c) {
-        return __builtin_mul_overflow(a, b, c);
-    }
-
-    int main() {
-        int out;
-        if (foo(1, 2, &out)) {
-            return 0;
-        }
-
-        return 0;
-    }" AWS_HAVE_GCC_OVERFLOW_MATH_EXTENSIONS)
-
-    if (USE_CPU_EXTENSIONS)
-        check_c_source_runs("
-        int main() {
-        int foo = 42;
-        _mulx_u32(1, 2, &foo);
-        return foo != 2;
-        }" AWS_HAVE_MSVC_MULX)
-    endif()
-
-endif()
-
-check_c_source_compiles("
-    #include <Windows.h>
-    #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
-    int main() {
-        return 0;
-    }
-    #else
-    it's not windows desktop
-    #endif
-" AWS_HAVE_WINAPI_DESKTOP)
-
-check_c_source_compiles("
-    int main() {
-#if !(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
-#    error \"not intel\"
-#endif
-        return 0;
-    }
-" AWS_ARCH_INTEL)
-
-check_c_source_compiles("
-    int main() {
-#if !(defined(__aarch64__) || defined(_M_ARM64))
-#    error \"not arm64\"
-#endif
-        return 0;
-    }
-" AWS_ARCH_ARM64)
-
-check_c_source_compiles("
-    int main() {
-#if !(defined(__arm__) || defined(_M_ARM))
-#    error \"not arm\"
-#endif
-        return 0;
-    }
-" AWS_ARCH_ARM32)
-
-check_c_source_compiles("
-int main() {
-    int foo = 42, bar = 24;
-    __asm__ __volatile__(\"\":\"=r\"(foo):\"r\"(bar):\"memory\");
-}" AWS_HAVE_GCC_INLINE_ASM)
-
-check_c_source_compiles("
-#include <sys/auxv.h>
-int main() {
-#ifdef __linux__
-    getauxval(AT_HWCAP);
-    getauxval(AT_HWCAP2);
-#endif
-    return 0;
-}" AWS_HAVE_AUXV)
-
-string(REGEX MATCH "^(aarch64|arm)" ARM_CPU "${CMAKE_SYSTEM_PROCESSOR}")
-if(NOT LEGACY_COMPILER_SUPPORT OR ARM_CPU)
-    check_c_source_compiles("
-    #include <execinfo.h>
-    int main() {
-        backtrace(NULL, 0);
-        return 0;
-    }" AWS_HAVE_EXECINFO)
-endif()
-
-check_c_source_compiles("
-#include <linux/if_link.h>
-int main() {
-    return 1;
-}" AWS_HAVE_LINUX_IF_LINK_H)
+set (AWS_HAVE_GCC_INLINE_ASM 1)
+set (AWS_HAVE_AUXV 1)

contrib/aws-cmake/AwsSIMD.cmake

@@ -1,54 +1,13 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
 
-include(CheckCCompilerFlag)
-include(CheckIncludeFile)
-
 if (USE_CPU_EXTENSIONS)
-    if (MSVC)
-        check_c_compiler_flag("/arch:AVX2" HAVE_M_AVX2_FLAG)
-        if (HAVE_M_AVX2_FLAG)
-            set(AVX2_CFLAGS "/arch:AVX2")
-        endif()
-    else()
-        check_c_compiler_flag(-mavx2 HAVE_M_AVX2_FLAG)
-        if (HAVE_M_AVX2_FLAG)
-            set(AVX2_CFLAGS "-mavx -mavx2")
-        endif()
+    if (HAVE_AVX2)
+        set (AVX2_CFLAGS "-mavx -mavx2")
+        set (HAVE_AVX2_INTRINSICS 1)
+        set (HAVE_MM256_EXTRACT_EPI64 1)
     endif()
-
-
-    cmake_push_check_state()
-    set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${AVX2_CFLAGS}")
-
-    check_c_source_compiles("
-        #include <immintrin.h>
-        #include <emmintrin.h>
-        #include <string.h>
-
-        int main() {
-            __m256i vec;
-            memset(&vec, 0, sizeof(vec));
-
-            _mm256_shuffle_epi8(vec, vec);
-            _mm256_set_epi32(1,2,3,4,5,6,7,8);
-            _mm256_permutevar8x32_epi32(vec, vec);
-
-            return 0;
-        }"  HAVE_AVX2_INTRINSICS)
-
-    check_c_source_compiles("
-        #include <immintrin.h>
-        #include <string.h>
-
-        int main() {
-            __m256i vec;
-            memset(&vec, 0, sizeof(vec));
-            return (int)_mm256_extract_epi64(vec, 2);
-        }" HAVE_MM256_EXTRACT_EPI64)
-
-    cmake_pop_check_state()
-endif() # USE_CPU_EXTENSIONS
+endif()
 
 macro(simd_add_definition_if target definition)
     if(${definition})

contrib/aws-cmake/AwsThreadAffinity.cmake

@@ -1,50 +1,9 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
 
-include(CheckSymbolExists)
-
 # Check if the platform supports setting thread affinity
 # (important for hitting full NIC entitlement on NUMA architectures)
 function(aws_set_thread_affinity_method target)
-
-    # Non-POSIX, Android, and Apple platforms do not support thread affinity.
-    if (NOT UNIX OR ANDROID OR APPLE)
-        target_compile_definitions(${target} PRIVATE
-            -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE)
-        return()
-    endif()
-
-    cmake_push_check_state()
-    list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE)
-    list(APPEND CMAKE_REQUIRED_LIBRARIES pthread)
-
-    set(headers "pthread.h")
-    # BSDs put nonportable pthread declarations in a separate header.
-    if(CMAKE_SYSTEM_NAME MATCHES BSD)
-        set(headers "${headers};pthread_np.h")
-    endif()
-
-    # Using pthread attrs is the preferred method, but is glibc-specific.
-    check_symbol_exists(pthread_attr_setaffinity_np "${headers}" USE_PTHREAD_ATTR_SETAFFINITY)
-    if (USE_PTHREAD_ATTR_SETAFFINITY)
-        target_compile_definitions(${target} PRIVATE
-            -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_PTHREAD_ATTR)
-        return()
-    endif()
-
-    # This method is still nonportable, but is supported by musl and BSDs.
-    check_symbol_exists(pthread_setaffinity_np "${headers}" USE_PTHREAD_SETAFFINITY)
-    if (USE_PTHREAD_SETAFFINITY)
-        target_compile_definitions(${target} PRIVATE
-            -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_PTHREAD)
-        return()
-    endif()
-
-    # If we got here, we expected thread affinity support but didn't find it.
-    # We still build with degraded NUMA performance, but show a warning.
-    message(WARNING "No supported method for setting thread affinity")
-    target_compile_definitions(${target} PRIVATE
-        -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE)
-
-    cmake_pop_check_state()
+    # This code has been cut, because I don't care about it.
+    target_compile_definitions(${target} PRIVATE -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE)
 endfunction()

contrib/aws-cmake/AwsThreadName.cmake

@@ -1,61 +1,13 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
 
-include(CheckSymbolExists)
-
 # Check how the platform supports setting thread name
 function(aws_set_thread_name_method target)
-
-    if (WINDOWS)
-        # On Windows we do a runtime check, instead of compile-time check
-        return()
-    elseif (APPLE)
+    if (APPLE)
         # All Apple platforms we support have the same function, so no need for compile-time check.
         return()
     endif()
 
-    cmake_push_check_state()
-    list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE)
-    list(APPEND CMAKE_REQUIRED_LIBRARIES pthread)
-
-    # The start of the test program
-    set(c_source_start "
-        #define _GNU_SOURCE
-        #include <pthread.h>
-
-        #if defined(__FreeBSD__) || defined(__NETBSD__)
-        #include <pthread_np.h>
-        #endif
-
-        int main() {
-            pthread_t thread_id;
-        ")
-
-    # The end of the test program
-    set(c_source_end "}")
-
     # pthread_setname_np() usually takes 2 args
-    check_c_source_compiles("
-        ${c_source_start}
-        pthread_setname_np(thread_id, \"asdf\");
-        ${c_source_end}"
-        PTHREAD_SETNAME_TAKES_2ARGS)
-    if (PTHREAD_SETNAME_TAKES_2ARGS)
-        target_compile_definitions(${target} PRIVATE -DAWS_PTHREAD_SETNAME_TAKES_2ARGS)
-        return()
-    endif()
-
-    # But on NetBSD it takes 3!
-    check_c_source_compiles("
-        ${c_source_start}
-        pthread_setname_np(thread_id, \"asdf\", NULL);
-        ${c_source_end}
-        " PTHREAD_SETNAME_TAKES_3ARGS)
-    if (PTHREAD_SETNAME_TAKES_3ARGS)
-        target_compile_definitions(${target} PRIVATE -DAWS_PTHREAD_SETNAME_TAKES_3ARGS)
-        return()
-    endif()
-
-    # And on many older/weirder platforms it's just not supported
-    cmake_pop_check_state()
+    target_compile_definitions(${target} PRIVATE -DAWS_PTHREAD_SETNAME_TAKES_2ARGS)
 endfunction()

contrib/google-protobuf

@@ -1 +1 @@
-Subproject commit 2a4fa1a4e95012d754ac55d43c8bc462dd1c78a8
+Subproject commit 089b89c8d4140f0d49fe4222b047a8ea814bc752

contrib/google-protobuf-cmake/CMakeLists.txt

@@ -20,7 +20,6 @@ endif()
 set(protobuf_source_dir "${ClickHouse_SOURCE_DIR}/contrib/google-protobuf")
 set(protobuf_binary_dir "${ClickHouse_BINARY_DIR}/contrib/google-protobuf")
 
-
 add_definitions(-DGOOGLE_PROTOBUF_CMAKE_BUILD)
 
 add_definitions(-DHAVE_PTHREAD)
@@ -30,17 +29,70 @@ include_directories(
   ${protobuf_binary_dir}
   ${protobuf_source_dir}/src)
 
+add_library(utf8_range
+    ${protobuf_source_dir}/third_party/utf8_range/naive.c
+    ${protobuf_source_dir}/third_party/utf8_range/range2-neon.c
+    ${protobuf_source_dir}/third_party/utf8_range/range2-sse.c
+)
+include_directories(${protobuf_source_dir}/third_party/utf8_range)
+
+add_library(utf8_validity
+    ${protobuf_source_dir}/third_party/utf8_range/utf8_validity.cc
+)
+target_link_libraries(utf8_validity PUBLIC absl::strings)
+
+set(protobuf_absl_used_targets
+    absl::absl_check
+    absl::absl_log
+    absl::algorithm
+    absl::base
+    absl::bind_front
+    absl::bits
+    absl::btree
+    absl::cleanup
+    absl::cord
+    absl::core_headers
+    absl::debugging
+    absl::die_if_null
+    absl::dynamic_annotations
+    absl::flags
+    absl::flat_hash_map
+    absl::flat_hash_set
+    absl::function_ref
+    absl::hash
+    absl::layout
+    absl::log_initialize
+    absl::log_severity
+    absl::memory
+    absl::node_hash_map
+    absl::node_hash_set
+    absl::optional
+    absl::span
+    absl::status
+    absl::statusor
+    absl::strings
+    absl::synchronization
+    absl::time
+    absl::type_traits
+    absl::utility
+    absl::variant
+)
+
 set(libprotobuf_lite_files
   ${protobuf_source_dir}/src/google/protobuf/any_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/arena.cc
+  ${protobuf_source_dir}/src/google/protobuf/arena_align.cc
+  ${protobuf_source_dir}/src/google/protobuf/arena_config.cc
   ${protobuf_source_dir}/src/google/protobuf/arenastring.cc
+  ${protobuf_source_dir}/src/google/protobuf/arenaz_sampler.cc
   ${protobuf_source_dir}/src/google/protobuf/extension_set.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_enum_util.cc
+  ${protobuf_source_dir}/src/google/protobuf/generated_message_tctable_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_message_util.cc
   ${protobuf_source_dir}/src/google/protobuf/implicit_weak_message.cc
+  ${protobuf_source_dir}/src/google/protobuf/inlined_string_field.cc
   ${protobuf_source_dir}/src/google/protobuf/io/coded_stream.cc
   ${protobuf_source_dir}/src/google/protobuf/io/io_win32.cc
-  ${protobuf_source_dir}/src/google/protobuf/io/strtod.cc
   ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream.cc
   ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl.cc
   ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl_lite.cc
@@ -48,21 +100,15 @@ set(libprotobuf_lite_files
   ${protobuf_source_dir}/src/google/protobuf/message_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/parse_context.cc
   ${protobuf_source_dir}/src/google/protobuf/repeated_field.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/bytestream.cc
+  ${protobuf_source_dir}/src/google/protobuf/repeated_ptr_field.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/common.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/int128.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/status.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/statusor.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/stringpiece.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/stringprintf.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/structurally_valid.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/strutil.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/time.cc
   ${protobuf_source_dir}/src/google/protobuf/wire_format_lite.cc
 )
 
 add_library(_libprotobuf-lite ${libprotobuf_lite_files})
-target_link_libraries(_libprotobuf-lite pthread)
+target_link_libraries(_libprotobuf-lite
+    pthread
+    utf8_validity)
 if(${CMAKE_SYSTEM_NAME} STREQUAL "Android")
     target_link_libraries(_libprotobuf-lite log)
 endif()
@@ -71,67 +117,89 @@ add_library(protobuf::libprotobuf-lite ALIAS _libprotobuf-lite)
 
 
 set(libprotobuf_files
-  ${protobuf_source_dir}/src/google/protobuf/any.cc
   ${protobuf_source_dir}/src/google/protobuf/any.pb.cc
   ${protobuf_source_dir}/src/google/protobuf/api.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/duration.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/empty.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/field_mask.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/source_context.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/struct.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/timestamp.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/type.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/wrappers.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/any.cc
+  ${protobuf_source_dir}/src/google/protobuf/any_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/arena.cc
+  ${protobuf_source_dir}/src/google/protobuf/arena_align.cc
+  ${protobuf_source_dir}/src/google/protobuf/arena_config.cc
+  ${protobuf_source_dir}/src/google/protobuf/arenastring.cc
+  ${protobuf_source_dir}/src/google/protobuf/arenaz_sampler.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/importer.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/parser.cc
   ${protobuf_source_dir}/src/google/protobuf/descriptor.cc
   ${protobuf_source_dir}/src/google/protobuf/descriptor.pb.cc
   ${protobuf_source_dir}/src/google/protobuf/descriptor_database.cc
-  ${protobuf_source_dir}/src/google/protobuf/duration.pb.cc
   ${protobuf_source_dir}/src/google/protobuf/dynamic_message.cc
-  ${protobuf_source_dir}/src/google/protobuf/empty.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/extension_set.cc
   ${protobuf_source_dir}/src/google/protobuf/extension_set_heavy.cc
-  ${protobuf_source_dir}/src/google/protobuf/field_mask.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/generated_enum_util.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_message_bases.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_message_reflection.cc
+  ${protobuf_source_dir}/src/google/protobuf/generated_message_tctable_full.cc
+  ${protobuf_source_dir}/src/google/protobuf/generated_message_tctable_gen.cc
+  ${protobuf_source_dir}/src/google/protobuf/generated_message_tctable_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/generated_message_util.cc
+  ${protobuf_source_dir}/src/google/protobuf/implicit_weak_message.cc
+  ${protobuf_source_dir}/src/google/protobuf/inlined_string_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/coded_stream.cc
   ${protobuf_source_dir}/src/google/protobuf/io/gzip_stream.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/io_win32.cc
   ${protobuf_source_dir}/src/google/protobuf/io/printer.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/strtod.cc
   ${protobuf_source_dir}/src/google/protobuf/io/tokenizer.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_sink.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/json/internal/lexer.cc
+  ${protobuf_source_dir}/src/google/protobuf/json/internal/message_path.cc
+  ${protobuf_source_dir}/src/google/protobuf/json/internal/parser.cc
+  ${protobuf_source_dir}/src/google/protobuf/json/internal/unparser.cc
+  ${protobuf_source_dir}/src/google/protobuf/json/internal/untyped_message.cc
+  ${protobuf_source_dir}/src/google/protobuf/json/internal/writer.cc
+  ${protobuf_source_dir}/src/google/protobuf/json/internal/zero_copy_buffered_stream.cc
+  ${protobuf_source_dir}/src/google/protobuf/json/json.cc
+  ${protobuf_source_dir}/src/google/protobuf/map.cc
   ${protobuf_source_dir}/src/google/protobuf/map_field.cc
   ${protobuf_source_dir}/src/google/protobuf/message.cc
+  ${protobuf_source_dir}/src/google/protobuf/message_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/parse_context.cc
   ${protobuf_source_dir}/src/google/protobuf/reflection_ops.cc
+  ${protobuf_source_dir}/src/google/protobuf/repeated_field.cc
   ${protobuf_source_dir}/src/google/protobuf/repeated_ptr_field.cc
   ${protobuf_source_dir}/src/google/protobuf/service.cc
-  ${protobuf_source_dir}/src/google/protobuf/source_context.pb.cc
-  ${protobuf_source_dir}/src/google/protobuf/struct.pb.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/substitute.cc
+  ${protobuf_source_dir}/src/google/protobuf/stubs/common.cc
   ${protobuf_source_dir}/src/google/protobuf/text_format.cc
-  ${protobuf_source_dir}/src/google/protobuf/timestamp.pb.cc
-  ${protobuf_source_dir}/src/google/protobuf/type.pb.cc
   ${protobuf_source_dir}/src/google/protobuf/unknown_field_set.cc
   ${protobuf_source_dir}/src/google/protobuf/util/delimited_message_util.cc
   ${protobuf_source_dir}/src/google/protobuf/util/field_comparator.cc
   ${protobuf_source_dir}/src/google/protobuf/util/field_mask_util.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/internal/datapiece.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/internal/default_value_objectwriter.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/internal/error_listener.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/internal/field_mask_utility.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/internal/json_escaping.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/internal/json_objectwriter.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/internal/json_stream_parser.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/internal/object_writer.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/internal/proto_writer.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/internal/protostream_objectsource.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/internal/protostream_objectwriter.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/internal/type_info.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/internal/type_info_test_helper.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/internal/utility.cc
-  ${protobuf_source_dir}/src/google/protobuf/util/json_util.cc
   ${protobuf_source_dir}/src/google/protobuf/util/message_differencer.cc
   ${protobuf_source_dir}/src/google/protobuf/util/time_util.cc
   ${protobuf_source_dir}/src/google/protobuf/util/type_resolver_util.cc
   ${protobuf_source_dir}/src/google/protobuf/wire_format.cc
-  ${protobuf_source_dir}/src/google/protobuf/wrappers.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/wire_format_lite.cc
 )
 
 add_library(_libprotobuf ${libprotobuf_lite_files} ${libprotobuf_files})
 if (ENABLE_FUZZING)
     target_compile_options(_libprotobuf PRIVATE "-fsanitize-recover=all")
 endif()
-target_link_libraries(_libprotobuf pthread)
-target_link_libraries(_libprotobuf ch_contrib::zlib)
+target_link_libraries(_libprotobuf
+    pthread
+    ch_contrib::zlib
+    utf8_validity
+    ${protobuf_absl_used_targets})
 if(${CMAKE_SYSTEM_NAME} STREQUAL "Android")
     target_link_libraries(_libprotobuf log)
 endif()
@@ -143,20 +211,21 @@ set(libprotoc_files
   ${protobuf_source_dir}/src/google/protobuf/compiler/code_generator.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/command_line_interface.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/enum.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/enum_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/extension.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field_generators/map_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field_generators/message_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field_generators/string_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/file.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/generator.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/helpers.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/map_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/message.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/message_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/padding_optimizer.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/parse_function_generator.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/primitive_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/service.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/string_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/tracker.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_doc_comment.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_enum.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_enum_field.cc
@@ -173,6 +242,7 @@ set(libprotoc_files
   ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_source_generator_base.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_wrapper_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/names.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/java/context.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/java/doc_comment.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/java/enum.cc
@@ -195,38 +265,48 @@ set(libprotoc_files
   ${protobuf_source_dir}/src/google/protobuf/compiler/java/message_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/java/message_field_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/java/message_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/message_serialization.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/java/name_resolver.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/names.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/java/primitive_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/java/primitive_field_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/java/service.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/java/shared_code_generator.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/java/string_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/java/string_field_lite.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_enum.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_enum_field.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_extension.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_field.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_file.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_generator.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_helpers.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_map_field.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_message.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_message_field.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_oneof.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_primitive_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/enum.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/enum_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/extension.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/file.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/helpers.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/import_writer.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/line_consumer.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/map_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/message.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/message_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/names.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/oneof.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/primitive_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/text_format_decode_data.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/php/names.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/php/php_generator.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/plugin.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/plugin.pb.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/python/generator.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/python/helpers.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/python/pyi_generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/retention.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/ruby/ruby_generator.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/subprocess.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/zip_writer.cc
 )
 
 add_library(_libprotoc ${libprotoc_files})
-target_link_libraries(_libprotoc _libprotobuf)
+target_link_libraries(_libprotoc
+    _libprotobuf
+    ${protobuf_absl_used_targets})
 add_library(protobuf::libprotoc ALIAS _libprotoc)
 
 set(protoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/main.cc)
@@ -235,7 +315,11 @@ if (CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME
     AND CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR)
 
     add_executable(protoc ${protoc_files})
-    target_link_libraries(protoc _libprotoc _libprotobuf pthread)
+    target_link_libraries(protoc _libprotoc
+        _libprotobuf
+        pthread
+        utf8_validity
+        ${protobuf_absl_used_targets})
     add_executable(protobuf::protoc ALIAS protoc)
 
     if (ENABLE_FUZZING)
@@ -255,6 +339,8 @@ else ()
 
         # This is quite ugly but I cannot make dependencies work propery.
 
+        set(abseil_source_dir "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp")
+
         execute_process(
             COMMAND mkdir -p ${PROTOC_BUILD_DIR}
             COMMAND_ECHO STDOUT)
@@ -269,6 +355,8 @@ else ()
                 "-Dprotobuf_BUILD_CONFORMANCE=0"
                 "-Dprotobuf_BUILD_EXAMPLES=0"
                 "-Dprotobuf_BUILD_PROTOC_BINARIES=1"
+                "-DABSL_ROOT_DIR=${abseil_source_dir}"
+                "-DABSL_ENABLE_INSTALL=0"
                 "${protobuf_source_dir}/cmake"
             WORKING_DIRECTORY "${PROTOC_BUILD_DIR}"
             COMMAND_ECHO STDOUT)

contrib/grpc

@@ -1 +1 @@
-Subproject commit b723ecae0991bb873fe87a595dfb187178733fde
+Subproject commit a08fe1a34075c93bb2d606dd608b9a3953288b81

contrib/libcxx-cmake/CMakeLists.txt

@@ -1,5 +1,3 @@
-include(CheckCXXCompilerFlag)
-
 set(LIBCXX_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm-project/libcxx")
 
 set(SRCS

contrib/libunwind-cmake/CMakeLists.txt

@@ -1,6 +1,3 @@
-include(CheckCCompilerFlag)
-include(CheckCXXCompilerFlag)
-
 set(LIBUNWIND_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libunwind")
 
 set(LIBUNWIND_CXX_SOURCES
@@ -48,27 +45,11 @@ target_compile_definitions(unwind PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIB
 # and disable sanitizers (otherwise infinite loop may happen)
 target_compile_options(unwind PRIVATE -O3 -fno-exceptions -funwind-tables -fno-sanitize=all $<$<COMPILE_LANGUAGE:CXX>:-nostdinc++ -fno-rtti>)
 
-check_c_compiler_flag(-Wunused-but-set-variable HAVE_WARNING_UNUSED_BUT_SET_VARIABLE)
-if (HAVE_WARNING_UNUSED_BUT_SET_VARIABLE)
-    target_compile_options(unwind PRIVATE -Wno-unused-but-set-variable)
-endif ()
-
-check_cxx_compiler_flag(-Wmissing-attributes HAVE_WARNING_MISSING_ATTRIBUTES)
-if (HAVE_WARNING_MISSING_ATTRIBUTES)
-    target_compile_options(unwind PRIVATE -Wno-missing-attributes)
-endif ()
-
-check_cxx_compiler_flag(-Wmaybe-uninitialized HAVE_WARNING_MAYBE_UNINITIALIZED)
-if (HAVE_WARNING_MAYBE_UNINITIALIZED)
-    target_compile_options(unwind PRIVATE -Wno-maybe-uninitialized)
-endif ()
+target_compile_options(unwind PRIVATE -Wno-unused-but-set-variable)
 
 # The library is using register variables that are bound to specific registers
 # Example: DwarfInstructions.hpp: register unsigned long long x16 __asm("x16") = cfa;
-check_cxx_compiler_flag(-Wregister HAVE_WARNING_REGISTER)
-if (HAVE_WARNING_REGISTER)
-    target_compile_options(unwind PRIVATE "$<$<COMPILE_LANGUAGE:CXX>:-Wno-register>")
-endif ()
+target_compile_options(unwind PRIVATE "$<$<COMPILE_LANGUAGE:CXX>:-Wno-register>")
 
 install(
     TARGETS unwind

contrib/rocksdb-cmake/CMakeLists.txt

@@ -76,7 +76,6 @@ else()
   endif()
 endif()
 
-include(CheckCCompilerFlag)
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
   if(POWER9)
     set(HAS_POWER9 1)
@@ -88,21 +87,12 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
 endif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm64|ARM64")
-  CHECK_C_COMPILER_FLAG("-march=armv8-a+crc+crypto" HAS_ARMV8_CRC)
-  if(HAS_ARMV8_CRC)
-    message(STATUS " HAS_ARMV8_CRC yes")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
-  endif(HAS_ARMV8_CRC)
+  set(HAS_ARMV8_CRC 1)
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
 endif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm64|ARM64")
 
 
-include(CheckCXXSourceCompiles)
-if(NOT MSVC)
-  set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul")
-endif()
-
-unset(CMAKE_REQUIRED_FLAGS)
 if(HAVE_SSE42)
   add_definitions(-DHAVE_SSE42)
   add_definitions(-DHAVE_PCLMUL)
@@ -121,75 +111,18 @@ elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
   add_definitions(-DOS_LINUX)
 elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS")
   add_definitions(-DOS_SOLARIS)
-elseif(CMAKE_SYSTEM_NAME MATCHES "kFreeBSD")
-  add_definitions(-DOS_GNU_KFREEBSD)
 elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
   add_definitions(-DOS_FREEBSD)
-elseif(CMAKE_SYSTEM_NAME MATCHES "NetBSD")
-  add_definitions(-DOS_NETBSD)
-elseif(CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
-  add_definitions(-DOS_OPENBSD)
-elseif(CMAKE_SYSTEM_NAME MATCHES "DragonFly")
-  add_definitions(-DOS_DRAGONFLYBSD)
 elseif(CMAKE_SYSTEM_NAME MATCHES "Android")
   add_definitions(-DOS_ANDROID)
-elseif(CMAKE_SYSTEM_NAME MATCHES "Windows")
-  add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX)
-  if(MINGW)
-    add_definitions(-D_WIN32_WINNT=_WIN32_WINNT_VISTA)
-  endif()
 endif()
 
-if(NOT WIN32)
-  add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX)
-endif()
+add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX)
 
-option(WITH_FALLOCATE "build with fallocate" ON)
-if(WITH_FALLOCATE)
-  CHECK_C_SOURCE_COMPILES("
-#include <fcntl.h>
-#include <linux/falloc.h>
-int main() {
- int fd = open(\"/dev/null\", 0);
- fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, 1024);
-}
-" HAVE_FALLOCATE)
-  if(HAVE_FALLOCATE)
-    add_definitions(-DROCKSDB_FALLOCATE_PRESENT)
-  endif()
-endif()
-
-CHECK_C_SOURCE_COMPILES("
-#include <fcntl.h>
-int main() {
-  int fd = open(\"/dev/null\", 0);
-  sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
-}
-" HAVE_SYNC_FILE_RANGE_WRITE)
-if(HAVE_SYNC_FILE_RANGE_WRITE)
-  add_definitions(-DROCKSDB_RANGESYNC_PRESENT)
-endif()
-
-CHECK_C_SOURCE_COMPILES("
-#include <pthread.h>
-int main() {
-  (void) PTHREAD_MUTEX_ADAPTIVE_NP;
-}
-" HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
-if(HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
+if (OS_LINUX OR OS_FREEBSD)
   add_definitions(-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX)
 endif()
 
-include(CheckCXXSymbolExists)
-if (OS_FREEBSD)
-  check_cxx_symbol_exists(malloc_usable_size "${ROCKSDB_SOURCE_DIR}/malloc_np.h" HAVE_MALLOC_USABLE_SIZE)
-else()
-  check_cxx_symbol_exists(malloc_usable_size "${ROCKSDB_SOURCE_DIR}/malloc.h" HAVE_MALLOC_USABLE_SIZE)
-endif()
-if(HAVE_MALLOC_USABLE_SIZE)
-  add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE)
-endif()
-
 if (OS_LINUX)
   add_definitions(-DROCKSDB_SCHED_GETCPU_PRESENT)
   add_definitions(-DROCKSDB_AUXV_SYSAUXV_PRESENT)
@@ -204,7 +137,6 @@ include_directories("${ROCKSDB_SOURCE_DIR}/include")
 if(WITH_FOLLY_DISTRIBUTED_MUTEX)
   include_directories("${ROCKSDB_SOURCE_DIR}/third-party/folly")
 endif()
-find_package(Threads REQUIRED)
 
 # Main library source code
 
@@ -497,7 +429,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/util/memarena.cc
     rocksdb_build_version.cc)
 
-if(HAVE_SSE42 AND NOT MSVC)
+if(HAVE_SSE42)
   set_source_files_properties(
     "${ROCKSDB_SOURCE_DIR}/util/crc32c.cc"
     PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")

contrib/xz-cmake/CMakeLists.txt

@@ -98,8 +98,6 @@ if (ARCH_S390X)
     add_compile_definitions(WORDS_BIGENDIAN)
 endif ()
 
-find_package(Threads REQUIRED)
-
 
 add_library(_liblzma
     ${SRC_DIR}/src/common/mythread.h

docker/docs/builder/Dockerfile

@@ -4,8 +4,8 @@ FROM node:16-alpine
 
 RUN apk add --no-cache git openssh bash
 
-# At this point we want to really update /opt/clickhouse-docs
-# despite the cached images
+# At this point we want to really update /opt/clickhouse-docs directory
+# So we reset the cache
 ARG CACHE_INVALIDATOR=0
 
 RUN git clone https://github.com/ClickHouse/clickhouse-docs.git \

docker/packager/binary/build.sh

@@ -177,11 +177,12 @@ then
     tar c -C /build/ --exclude='.git/modules/**' .git | tar x -C "$PERF_OUTPUT"/ch
     # Create branch pr and origin/master to have them for the following performance comparison
     git -C "$PERF_OUTPUT"/ch branch pr
-    git -C "$PERF_OUTPUT"/ch fetch --no-tags --depth 50 origin master:origin/master
+    git -C "$PERF_OUTPUT"/ch fetch --no-tags --no-recurse-submodules --depth 50 origin master:origin/master
     # Clean remote, to not have it stale
     git -C "$PERF_OUTPUT"/ch remote | xargs -n1 git -C "$PERF_OUTPUT"/ch remote remove
     # And clean all tags
-    git -C "$PERF_OUTPUT"/ch tag | xargs git -C "$PERF_OUTPUT"/ch tag -d
+    echo "Deleting $(git -C "$PERF_OUTPUT"/ch tag | wc -l) tags"
+    git -C "$PERF_OUTPUT"/ch tag | xargs git -C "$PERF_OUTPUT"/ch tag -d >/dev/null
     git -C "$PERF_OUTPUT"/ch reset --soft pr
     git -C "$PERF_OUTPUT"/ch log -5
     (

docker/test/integration/mysql_java_client/MySQLJavaClientTest.java

@@ -39,8 +39,7 @@ public class MySQLJavaClientTest {
 
         // useServerPrepStmts=true -> COM_STMT_PREPARE + COM_STMT_EXECUTE -> binary
         // useServerPrepStmts=false -> COM_QUERY -> text
-        String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?useSSL=false&useServerPrepStmts=%s",
-                host, port, database, binary);
+        String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?useSSL=false&useServerPrepStmts=%s", host, port, database, binary);
 
         try {
             Class.forName("com.mysql.cj.jdbc.Driver");
@@ -67,21 +66,21 @@ public class MySQLJavaClientTest {
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
-            System.out.printf("%s, value: %d\n", getMysqlType(rs, "i8"), rs.getInt("i8"));
-            System.out.printf("%s, value: %d\n", getMysqlType(rs, "i16"), rs.getInt("i16"));
-            System.out.printf("%s, value: %d\n", getMysqlType(rs, "i32"), rs.getInt("i32"));
-            System.out.printf("%s, value: %d\n", getMysqlType(rs, "i64"), rs.getLong("i64"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "i128"), rs.getString("i128"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "i256"), rs.getString("i256"));
-            System.out.printf("%s, value: %d\n", getMysqlType(rs, "ui8"), rs.getInt("ui8"));
-            System.out.printf("%s, value: %d\n", getMysqlType(rs, "ui16"), rs.getInt("ui16"));
-            System.out.printf("%s, value: %d\n", getMysqlType(rs, "ui32"), rs.getLong("ui32"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "ui64"), rs.getString("ui64"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "ui128"), rs.getString("ui128"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "ui256"), rs.getString("ui256"));
-            System.out.printf("%s, value: %f\n", getMysqlType(rs, "f32"), rs.getFloat("f32"));
-            System.out.printf("%s, value: %f\n", getMysqlType(rs, "f64"), rs.getFloat("f64"));
-            System.out.printf("%s, value: %b\n", getMysqlType(rs, "b"), rs.getBoolean("b"));
+            System.out.printf("%s, value: %d, wasNull: %b\n", getMysqlType(rs, "i8"), rs.getInt("i8"), rs.wasNull());
+            System.out.printf("%s, value: %d, wasNull: %b\n", getMysqlType(rs, "i16"), rs.getInt("i16"), rs.wasNull());
+            System.out.printf("%s, value: %d, wasNull: %b\n", getMysqlType(rs, "i32"), rs.getInt("i32"), rs.wasNull());
+            System.out.printf("%s, value: %d, wasNull: %b\n", getMysqlType(rs, "i64"), rs.getLong("i64"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "i128"), rs.getString("i128"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "i256"), rs.getString("i256"), rs.wasNull());
+            System.out.printf("%s, value: %d, wasNull: %b\n", getMysqlType(rs, "ui8"), rs.getInt("ui8"), rs.wasNull());
+            System.out.printf("%s, value: %d, wasNull: %b\n", getMysqlType(rs, "ui16"), rs.getInt("ui16"), rs.wasNull());
+            System.out.printf("%s, value: %d, wasNull: %b\n", getMysqlType(rs, "ui32"), rs.getLong("ui32"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "ui64"), rs.getString("ui64"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "ui128"), rs.getString("ui128"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "ui256"), rs.getString("ui256"), rs.wasNull());
+            System.out.printf("%s, value: %f, wasNull: %b\n", getMysqlType(rs, "f32"), rs.getFloat("f32"), rs.wasNull());
+            System.out.printf("%s, value: %f, wasNull: %b\n", getMysqlType(rs, "f64"), rs.getFloat("f64"), rs.wasNull());
+            System.out.printf("%s, value: %b, wasNull: %b\n", getMysqlType(rs, "b"), rs.getBoolean("b"), rs.wasNull());
         }
         System.out.println();
     }
@@ -92,10 +91,10 @@ public class MySQLJavaClientTest {
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "s"), rs.getString("s"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "sn"), rs.getString("sn"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "lc"), rs.getString("lc"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "nlc"), rs.getString("nlc"));
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "s"), rs.getString("s"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "sn"), rs.getString("sn"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "lc"), rs.getString("lc"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "nlc"), rs.getString("nlc"), rs.wasNull());
         }
         System.out.println();
     }
@@ -106,10 +105,10 @@ public class MySQLJavaClientTest {
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "ilc"), rs.getInt("ilc"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dlc"), rs.getDate("dlc"));
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "ilc"), rs.getInt("ilc"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dlc"), rs.getDate("dlc"), rs.wasNull());
             // NULL int is represented as zero
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "ni"), rs.getInt("ni"));
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "ni"), rs.getInt("ni"), rs.wasNull());
         }
         System.out.println();
     }
@@ -120,12 +119,11 @@ public class MySQLJavaClientTest {
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "d32"), rs.getBigDecimal("d32").toPlainString());
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "d64"), rs.getBigDecimal("d64").toPlainString());
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "d128_native"),
-                    rs.getBigDecimal("d128_native").toPlainString());
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "d128_text"), rs.getString("d128_text"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "d256"), rs.getString("d256"));
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "d32"), rs.getBigDecimal("d32").toPlainString(), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "d64"), rs.getBigDecimal("d64").toPlainString(), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "d128_native"), rs.getBigDecimal("d128_native").toPlainString(), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "d128_text"), rs.getString("d128_text"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "d256"), rs.getString("d256"), rs.wasNull());
         }
         System.out.println();
     }
@@ -136,12 +134,12 @@ public class MySQLJavaClientTest {
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "d"), rs.getDate("d"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "d32"), rs.getDate("d32"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt"), rs.getTimestamp("dt"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt64_3"), rs.getTimestamp("dt64_3"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt64_6"), rs.getTimestamp("dt64_6"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt64_9"), rs.getTimestamp("dt64_9"));
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "d"), rs.getDate("d"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "d32"), rs.getDate("d32"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dt"), rs.getTimestamp("dt"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dt64_3"), rs.getTimestamp("dt64_3"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dt64_6"), rs.getTimestamp("dt64_6"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dt64_9"), rs.getTimestamp("dt64_9"), rs.wasNull());
         }
         System.out.println();
     }
@@ -152,13 +150,13 @@ public class MySQLJavaClientTest {
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt64_0"), rs.getTimestamp("dt64_0"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt64_1"), rs.getTimestamp("dt64_1"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt64_2"), rs.getTimestamp("dt64_2"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt64_4"), rs.getTimestamp("dt64_4"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt64_5"), rs.getTimestamp("dt64_5"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt64_7"), rs.getTimestamp("dt64_7"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt64_8"), rs.getTimestamp("dt64_8"));
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dt64_0"), rs.getTimestamp("dt64_0"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dt64_1"), rs.getTimestamp("dt64_1"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dt64_2"), rs.getTimestamp("dt64_2"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dt64_4"), rs.getTimestamp("dt64_4"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dt64_5"), rs.getTimestamp("dt64_5"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dt64_7"), rs.getTimestamp("dt64_7"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dt64_8"), rs.getTimestamp("dt64_8"), rs.wasNull());
         }
         System.out.println();
     }
@@ -169,8 +167,8 @@ public class MySQLJavaClientTest {
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt"), rs.getTimestamp("dt"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt64_3"), rs.getTimestamp("dt64_3"));
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dt"), rs.getTimestamp("dt"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dt64_3"), rs.getTimestamp("dt64_3"), rs.wasNull());
         }
         System.out.println();
     }
@@ -181,10 +179,10 @@ public class MySQLJavaClientTest {
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "a"), rs.getString("a"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "u"), rs.getString("u"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "t"), rs.getString("t"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "m"), rs.getString("m"));
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "a"), rs.getString("a"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "u"), rs.getString("u"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "t"), rs.getString("t"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "m"), rs.getString("m"), rs.wasNull());
         }
         System.out.println();
     }
@@ -196,17 +194,15 @@ public class MySQLJavaClientTest {
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "f"), rs.getFloat("f"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "d"), rs.getDate("d"));
-            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt"), rs.getTimestamp("dt"));
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "f"), rs.getFloat("f"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "d"), rs.getDate("d"), rs.wasNull());
+            System.out.printf("%s, value: %s, wasNull: %b\n", getMysqlType(rs, "dt"), rs.getTimestamp("dt"), rs.wasNull());
         }
         System.out.println();
     }
 
     private static String getMysqlType(ResultSet rs, String columnLabel) throws SQLException {
         ResultSetMetaData meta = rs.getMetaData();
-        return String.format("%s type is %s", columnLabel,
-                MysqlType.getByJdbcType(meta.getColumnType(rs.findColumn(columnLabel))));
+        return String.format("%s type is %s", columnLabel, MysqlType.getByJdbcType(meta.getColumnType(rs.findColumn(columnLabel))));
     }
-
 }

docker/test/integration/runner/Dockerfile

@@ -68,6 +68,7 @@ RUN python3 -m pip install --no-cache-dir \
     asyncio \
     avro==1.10.2 \
     azure-storage-blob \
+    boto3 \
     cassandra-driver \
     confluent-kafka==1.9.2 \
     delta-spark==2.3.0 \

docker/test/stateless/stress_tests.lib

@@ -53,7 +53,7 @@ function configure()
       > /etc/clickhouse-server/config.d/keeper_port.xml.tmp
     sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
 
-    function randomize_keeper_config_boolean_value {
+    function randomize_config_boolean_value {
         value=$(($RANDOM % 2))
         sudo cat /etc/clickhouse-server/config.d/$2.xml \
         | sed "s|<$1>[01]</$1>|<$1>$value</$1>|" \
@@ -72,7 +72,11 @@ function configure()
     sudo chown clickhouse /etc/clickhouse-server/config.d/keeper_port.xml
     sudo chgrp clickhouse /etc/clickhouse-server/config.d/keeper_port.xml
 
-    randomize_config_boolean_value use_compression zookeeper
+    if [[ -n "$ZOOKEEPER_FAULT_INJECTION" ]] && [[ "$ZOOKEEPER_FAULT_INJECTION" -eq 1 ]]; then
+        randomize_config_boolean_value use_compression zookeeper_fault_injection
+    else
+        randomize_config_boolean_value use_compression zookeeper
+    fi
 
     # for clickhouse-server (via service)
     echo "ASAN_OPTIONS='malloc_context_size=10 verbosity=1 allocator_release_to_os_interval_ms=10000'" >> /etc/environment

docs/_includes/install/universal.sh

@@ -20,9 +20,9 @@ then
         fi
     elif [ "${ARCH}" = "aarch64" -o "${ARCH}" = "arm64" ]
     then
-        # If the system has >=ARMv8.2 (https://en.wikipedia.org/wiki/AArch64), choose the corresponding build, else fall back to a v8.0
-        # compat build. Unfortunately, the ARM ISA level cannot be read directly, we need to guess from the "features" in /proc/cpuinfo.
-        # Also, the flags in /proc/cpuinfo are named differently than the flags passed to the compiler (cmake/cpu_features.cmake).
+        # Dispatch between standard and compatibility builds, see cmake/cpu_features.cmake for details. Unfortunately, (1) the ARM ISA level
+        # cannot be read directly, we need to guess from the "features" in /proc/cpuinfo, and (2) the flags in /proc/cpuinfo are named
+        # differently than the flags passed to the compiler in cpu_features.cmake.
         HAS_ARMV82=$(grep -m 1 'Features' /proc/cpuinfo | awk '/asimd/ && /sha1/ && /aes/ && /atomics/ && /lrcpc/')
         if [ "${HAS_ARMV82}" ]
         then

docs/en/engines/table-engines/integrations/kafka.md

@@ -238,19 +238,19 @@ Example:
 
 ## Virtual Columns {#virtual-columns}
 
-- `_topic` — Kafka topic.
-- `_key` — Key of the message.
-- `_offset` — Offset of the message.
-- `_timestamp` — Timestamp of the message.
-- `_timestamp_ms` — Timestamp in milliseconds of the message.
-- `_partition` — Partition of Kafka topic.
-- `_headers.name` — Array of message's headers keys.
-- `_headers.value` — Array of message's headers values.
+- `_topic` — Kafka topic. Data type: `LowCardinality(String)`.
+- `_key` — Key of the message. Data type: `String`.
+- `_offset` — Offset of the message. Data type: `UInt64`.
+- `_timestamp` — Timestamp of the message Data type: `Nullable(DateTime)`.
+- `_timestamp_ms` — Timestamp in milliseconds of the message. Data type: `Nullable(DateTime64(3))`.
+- `_partition` — Partition of Kafka topic. Data type: `UInt64`.
+- `_headers.name` — Array of message's headers keys. Data type: `Array(String)`.
+- `_headers.value` — Array of message's headers values. Data type: `Array(String)`.
 
 Additional virtual columns when `kafka_handle_error_mode='stream'`:
 
-- `_raw_message` - Raw message that couldn't be parsed successfully.
-- `_error` - Exception message happened during failed parsing.
+- `_raw_message` - Raw message that couldn't be parsed successfully. Data type: `String`.
+- `_error` - Exception message happened during failed parsing. Data type: `String`.
 
 Note: `_raw_message` and `_error` virtual columns are filled only in case of exception during parsing, they are always empty when message was parsed successfully.
 

docs/en/engines/table-engines/integrations/nats.md

@@ -163,14 +163,14 @@ If you want to change the target table by using `ALTER`, we recommend disabling
 
 ## Virtual Columns {#virtual-columns}
 
-- `_subject` - NATS message subject.
+- `_subject` - NATS message subject. Data type: `String`.
 
 Additional virtual columns when `kafka_handle_error_mode='stream'`:
 
-- `_raw_message` - Raw message that couldn't be parsed successfully.
-- `_error` - Exception message happened during failed parsing.
+- `_raw_message` - Raw message that couldn't be parsed successfully. Data type: `Nullable(String)`.
+- `_error` - Exception message happened during failed parsing. Data type: `Nullable(String)`.
 
-Note: `_raw_message` and `_error` virtual columns are filled only in case of exception during parsing, they are always empty when message was parsed successfully.
+Note: `_raw_message` and `_error` virtual columns are filled only in case of exception during parsing, they are always `NULL` when message was parsed successfully.
 
 
 ## Data formats support {#data-formats-support}

docs/en/engines/table-engines/integrations/rabbitmq.md

@@ -184,19 +184,19 @@ Example:
 
 ## Virtual Columns {#virtual-columns}
 
-- `_exchange_name` - RabbitMQ exchange name.
-- `_channel_id` - ChannelID, on which consumer, who received the message, was declared.
-- `_delivery_tag` - DeliveryTag of the received message. Scoped per channel.
-- `_redelivered` - `redelivered` flag of the message.
-- `_message_id` - messageID of the received message; non-empty if was set, when message was published.
-- `_timestamp` - timestamp of the received message; non-empty if was set, when message was published.
+- `_exchange_name` - RabbitMQ exchange name. Data type: `String`.
+- `_channel_id` - ChannelID, on which consumer, who received the message, was declared. Data type: `String`.
+- `_delivery_tag` - DeliveryTag of the received message. Scoped per channel. Data type: `UInt64`.
+- `_redelivered` - `redelivered` flag of the message. Data type: `UInt8`.
+- `_message_id` - messageID of the received message; non-empty if was set, when message was published. Data type: `String`.
+- `_timestamp` - timestamp of the received message; non-empty if was set, when message was published. Data type: `UInt64`.
 
 Additional virtual columns when `kafka_handle_error_mode='stream'`:
 
-- `_raw_message` - Raw message that couldn't be parsed successfully.
-- `_error` - Exception message happened during failed parsing.
+- `_raw_message` - Raw message that couldn't be parsed successfully. Data type: `Nullable(String)`.
+- `_error` - Exception message happened during failed parsing. Data type: `Nullable(String)`.
 
-Note: `_raw_message` and `_error` virtual columns are filled only in case of exception during parsing, they are always empty when message was parsed successfully.
+Note: `_raw_message` and `_error` virtual columns are filled only in case of exception during parsing, they are always `NULL` when message was parsed successfully.
 
 ## Data formats support {#data-formats-support}
 

docs/en/engines/table-engines/special/filelog.md

@@ -94,12 +94,12 @@ If you want to change the target table by using `ALTER`, we recommend disabling
 
 ## Virtual Columns {#virtual-columns}
 
-- `_filename` - Name of the log file.
-- `_offset` - Offset in the log file.
+- `_filename` - Name of the log file. Data type: `LowCardinality(String)`.
+- `_offset` - Offset in the log file. Data type: `UInt64`.
 
 Additional virtual columns when `kafka_handle_error_mode='stream'`:
 
-- `_raw_record` - Raw record that couldn't be parsed successfully.
-- `_error` - Exception message happened during failed parsing.
+- `_raw_record` - Raw record that couldn't be parsed successfully. Data type: `Nullable(String)`.
+- `_error` - Exception message happened during failed parsing. Data type: `Nullable(String)`.
 
-Note: `_raw_record` and `_error` virtual columns are filled only in case of exception during parsing, they are always empty when message was parsed successfully.
+Note: `_raw_record` and `_error` virtual columns are filled only in case of exception during parsing, they are always `NULL` when message was parsed successfully.

docs/en/interfaces/formats.md

@@ -2469,6 +2469,7 @@ This function is designed to load a NumPy array from a .npy file into ClickHouse
 | u2       |    UInt16       |
 | u4       |    UInt32       |
 | u8       |    UInt64       |
+| f2       |    Float32      |
 | f4       |    Float32      |
 | f8       |    Float64      |
 | S        |    String       |

docs/en/operations/_troubleshooting.md

@@ -17,12 +17,8 @@
 
 - The issue may be happened when the GPG key is changed.
 
-Please use the following scripts to resolve the issue:
+Please use the manual from the [setup](../getting-started/install.md#setup-the-debian-repository) page to update the repository configuration.
 
-```bash
-sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
-sudo apt-get update
-```
 
 ### You Get Different Warnings with `apt-get update` {#you-get-different-warnings-with-apt-get-update}
 

docs/en/operations/query-cache.md

@@ -169,7 +169,12 @@ Also, results of queries with non-deterministic functions are not cached by defa
   [`getMacro()`](../sql-reference/functions/other-functions.md#getMacro) etc.
 
 To force caching of results of queries with non-deterministic functions regardless, use setting
-[query_cache_store_results_of_queries_with_nondeterministic_functions](settings/settings.md#query-cache-store-results-of-queries-with-nondeterministic-functions).
+[query_cache_nondeterministic_function_handling](settings/settings.md#query-cache-nondeterministic-function-handling).
+
+:::note
+Prior to ClickHouse v23.11, setting 'query_cache_store_results_of_queries_with_nondeterministic_functions = 0 / 1' controlled whether
+results of queries with non-deterministic results were cached. In newer ClickHouse versions, this setting is obsolete and has no effect.
+:::
 
 Finally, entries in the query cache are not shared between users due to security reasons. For example, user A must not be able to bypass a
 row policy on a table by running the same query as another user B for whom no such policy exists. However, if necessary, cache entries can

docs/en/operations/server-configuration-parameters/settings.md

@@ -961,9 +961,13 @@ See also “[Executable User Defined Functions](../../sql-reference/functions/in
 
 Lazy loading of dictionaries.
 
-If `true`, then each dictionary is created on first use. If dictionary creation failed, the function that was using the dictionary throws an exception.
+If `true`, then each dictionary is loaded on the first use. If the loading is failed, the function that was using the dictionary throws an exception.
 
-If `false`, all dictionaries are created when the server starts, if the dictionary or dictionaries are created too long or are created with errors, then the server boots without of these dictionaries and continues to try to create these dictionaries.
+If `false`, then the server starts loading all dictionaries at startup.
+Dictionaries are loaded in background.
+The server doesn't wait at startup until all the dictionaries finish their loading
+(exception: if `wait_dictionaries_load_at_startup` is set to `true` - see below).
+When a dictionary is used in a query for the first time then the query waits until the dictionary is loaded if it's not loaded yet.
 
 The default is `true`.
 
@@ -2391,6 +2395,24 @@ Path to the file that contains:
 <users_config>users.xml</users_config>
 ```
 
+## wait_dictionaries_load_at_startup {#wait_dictionaries_load_at_startup}
+
+If `false`, then the server will not wait at startup until all the dictionaries finish their loading.
+This allows to start ClickHouse faster.
+
+If `true`, then the server will wait at startup until all the dictionaries finish their loading (successfully or not)
+before listening to any connections.
+This can make ClickHouse start slowly, however after that some queries can be executed faster
+(because they won't have to wait for the used dictionaries to be load).
+
+The default is `false`.
+
+**Example**
+
+``` xml
+<wait_dictionaries_load_at_startup>false</wait_dictionaries_load_at_startup>
+```
+
 ## zookeeper {#server-settings_zookeeper}
 
 Contains settings that allow ClickHouse to interact with a [ZooKeeper](http://zookeeper.apache.org/) cluster.

docs/en/operations/settings/settings.md

@@ -1657,16 +1657,17 @@ Possible values:
 
 Default value: `1`.
 
-## query_cache_store_results_of_queries_with_nondeterministic_functions {#query-cache-store-results-of-queries-with-nondeterministic-functions}
+## query_cache_nondeterministic_function_handling {#query-cache-nondeterministic-function-handling}
 
-If turned on, then results of `SELECT` queries with non-deterministic functions (e.g. `rand()`, `now()`) can be cached in the [query cache](../query-cache.md).
+Controls how the [query cache](../query-cache.md) handles `SELECT` queries with non-deterministic functions like `rand()` or `now()`.
 
 Possible values:
 
-- 0 - Disabled
-- 1 - Enabled
+- `'throw'` - Throw an exception and don't cache the query result.
+- `'save'` - Cache the query result.
+- `'ignore'` - Don't cache the query result and don't throw an exception.
 
-Default value: `0`.
+Default value: `throw`.
 
 ## query_cache_min_query_runs {#query-cache-min-query-runs}
 

docs/en/operations/utilities/backupview.md

@@ -0,0 +1,50 @@
+---
+slug: /en/operations/utilities/backupview
+title: clickhouse_backupview
+---
+
+# clickhouse_backupview {#clickhouse_backupview}
+
+Python module to help analyzing backups made by the [BACKUP](https://clickhouse.com/docs/en/operations/backup) command.
+The main motivation was to allows getting some information from a backup without actually restoring it.
+
+This module provides functions to
+- enumerate files contained in a backup
+- read files from a backup
+- get useful information in readable form about databases, tables, parts contained in a backup
+- check integrity of a backup
+
+## Example:
+
+```python
+from clickhouse_backupview import open_backup, S3, FileInfo
+
+# Open a backup. We could also use a local path:
+# backup = open_backup("/backups/my_backup_1/")
+backup = open_backup(S3("uri", "access_key_id", "secret_access_key"))
+
+# Get a list of databasess inside the backup.
+print(backup.get_databases()))
+
+# Get a list of tables inside the backup,
+# and for each table its create query and a list of parts and partitions.
+for db in backup.get_databases():
+    for tbl in backup.get_tables(database=db):
+        print(backup.get_create_query(database=db, table=tbl))
+        print(backup.get_partitions(database=db, table=tbl))
+        print(backup.get_parts(database=db, table=tbl))
+
+# Extract everything from the backup.
+backup.extract_all(table="mydb.mytable", out='/tmp/my_backup_1/all/')
+
+# Extract the data of a specific table.
+backup.extract_table_data(table="mydb.mytable", out='/tmp/my_backup_1/mytable/')
+
+# Extract a single partition.
+backup.extract_table_data(table="mydb.mytable", partition="202201", out='/tmp/my_backup_1/202201/')
+
+# Extract a single part.
+backup.extract_table_data(table="mydb.mytable", part="202201_100_200_3", out='/tmp/my_backup_1/202201_100_200_3/')
+```
+
+For more examples see the [test](https://github.com/ClickHouse/ClickHouse/blob/master/utils/backupview/test/test.py).

docs/en/operations/utilities/index.md

@@ -16,3 +16,4 @@ pagination_next: 'en/operations/utilities/clickhouse-copier'
 - [clickhouse-disks](../../operations/utilities/clickhouse-disks.md) -- Provides filesystem-like operations
  on files among different ClickHouse disks.
 - [clickhouse-odbc-bridge](../../operations/utilities/odbc-bridge.md) — A proxy server for ODBC driver.
+- [clickhouse_backupview](../../operations/utilities/backupview.md) — A python module to analyze ClickHouse backups.

docs/en/sql-reference/functions/date-time-functions.md

@@ -1381,7 +1381,7 @@ toStartOfFifteenMinutes(toDateTime('2023-04-21 10:20:00')): 2023-04-21 10:15:00
 toStartOfFifteenMinutes(toDateTime('2023-04-21 10:23:00')): 2023-04-21 10:15:00
 ```
 
-## toStartOfInterval(time_or_data, INTERVAL x unit \[, time_zone\])
+## toStartOfInterval(date_or_date_with_time, INTERVAL x unit \[, time_zone\])
 
 This function generalizes other `toStartOf*()` functions. For example,
 - `toStartOfInterval(t, INTERVAL 1 year)` returns the same as `toStartOfYear(t)`,

docs/en/sql-reference/table-functions/file.md

@@ -1,4 +1,4 @@
----
+ --
 slug: /en/sql-reference/table-functions/file
 sidebar_position: 60
 sidebar_label: file
@@ -6,7 +6,7 @@ sidebar_label: file
 
 # file
 
-Provides a table-like interface to SELECT from and INSERT to files. This table function is similar to the [s3](/docs/en/sql-reference/table-functions/url.md) table function.  Use file() when working with local files, and s3() when working with buckets in S3, GCS, or MinIO.
+A table engine which provides a table-like interface to SELECT from and INSERT into files, similar to the [s3](/docs/en/sql-reference/table-functions/url.md) table function.  Use `file()` when working with local files, and `s3()` when working with buckets in object storage such as S3, GCS, or MinIO.
 
 The `file` function can be used in `SELECT` and `INSERT` queries to read from or write to files.
 
@@ -18,18 +18,18 @@ file([path_to_archive ::] path [,format] [,structure] [,compression])
 
 **Parameters**
 
-- `path` — The relative path to the file from [user_files_path](/docs/en/operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Path to file support following globs in read-only mode: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc', 'def'` — strings.
-- `path_to_archive` - The relative path to zip/tar/7z archive. Path to archive support the same globs as `path`.
+- `path` — The relative path to the file from [user_files_path](/docs/en/operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Supports in read-only mode the following [globs](#globs_in_path): `*`, `?`, `{abc,def}` (with `'abc'` and `'def'` being strings) and `{N..M}` (with `N` and `M` being numbers).
+- `path_to_archive` - The relative path to a zip/tar/7z archive. Supports the same globs as `path`.
 - `format` — The [format](/docs/en/interfaces/formats.md#formats) of the file.
 - `structure` — Structure of the table. Format: `'column1_name column1_type, column2_name column2_type, ...'`.
-- `compression` — The existing compression type when used in a `SELECT` query, or the desired compression type when used in an `INSERT` query.  The supported compression types are `gz`, `br`, `xz`, `zst`, `lz4`, and `bz2`.
+- `compression` — The existing compression type when used in a `SELECT` query, or the desired compression type when used in an `INSERT` query. Supported compression types are `gz`, `br`, `xz`, `zst`, `lz4`, and `bz2`.
 
 
 **Returned value**
 
-A table with the specified structure for reading or writing data in the specified file.
+A table for reading or writing data in a file.
 
-## File Write Examples
+## Examples for Writing to a File
 
 ### Write to a TSV file
 
@@ -48,9 +48,9 @@ As a result, the data is written into the file `test.tsv`:
 1	3	2
 ```
 
-### Partitioned Write to multiple TSV files
+### Partitioned write to multiple TSV files
 
-If you specify `PARTITION BY` expression when inserting data into a file() function, a separate file is created for each partition value. Splitting the data into separate files helps to improve reading operations efficiency.
+If you specify a `PARTITION BY` expression when inserting data into a table function of type `file()`, then a separate file is created for each partition. Splitting the data into separate files helps to improve performance of read operations.
 
 ```sql
 INSERT INTO TABLE FUNCTION
@@ -72,11 +72,11 @@ As a result, the data is written into three files: `test_1.tsv`, `test_2.tsv`, a
 1	2	3
 ```
 
-## File Read Examples
+## Examples for Reading from a File
 
 ### SELECT from a CSV file
 
-Setting `user_files_path` and the contents of the file `test.csv`:
+First, set `user_files_path` in the server configuration and prepare a file `test.csv`:
 
 ``` bash
 $ grep user_files_path /etc/clickhouse-server/config.xml
@@ -88,7 +88,7 @@ $ cat /var/lib/clickhouse/user_files/test.csv
     78,43,45
 ```
 
-Getting data from a table in `test.csv` and selecting the first two rows from it:
+Then, read data from `test.csv` into a table and select its first two rows:
 
 ``` sql
 SELECT * FROM
@@ -103,14 +103,6 @@ LIMIT 2;
 └─────────┴─────────┴─────────┘
 ```
 
-Getting the first 10 lines of a table that contains 3 columns of [UInt32](/docs/en/sql-reference/data-types/int-uint.md) type from a CSV file:
-
-``` sql
-SELECT * FROM
-file('test.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32')
-LIMIT 10;
-```
-
 ### Inserting data from a file into a table:
 
 ``` sql
@@ -130,41 +122,42 @@ file('test.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32');
 └─────────┴─────────┴─────────┘
 ```
 
-Getting data from table in table.csv, located in archive1.zip or/and archive2.zip
+Reading data from `table.csv`, located in `archive1.zip` or/and `archive2.zip`:
+
 ``` sql
 SELECT * FROM file('user_files/archives/archive{1..2}.zip :: table.csv');
 ```
 
-## Globs in Path {#globs_in_path}
+## Globbing {#globs_in_path}
 
-Multiple path components can have globs. For being processed file must exist and match to the whole path pattern (not only suffix or prefix).
+Paths may use globbing. Files must match the whole path pattern, not only the suffix or prefix.
 
-- `*` — Substitutes any number of any characters except `/` including empty string.
-- `?` — Substitutes any single character.
-- `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. The strings can contain the `/` symbol.
-- `{N..M}` — Substitutes any number in range from N to M including both borders.
-- `**` - Fetches all files inside the folder recursively.
+- `*` — Represents arbitrarily many characters except `/` but including the empty string.
+- `?` — Represents an arbitrary single character.
+- `{some_string,another_string,yet_another_one}` — Represents any of alternative strings `'some_string', 'another_string', 'yet_another_one'`. The strings may contain `/`.
+- `{N..M}` — Represents any number `>= N` and `<= M`.
+- `**` - Represents all files inside a folder recursively.
 
 Constructions with `{}` are similar to the [remote](remote.md) table function.
 
 **Example**
 
-Suppose we have several files with the following relative paths:
+Suppose there are these files with the following relative paths:
 
-- 'some_dir/some_file_1'
-- 'some_dir/some_file_2'
-- 'some_dir/some_file_3'
-- 'another_dir/some_file_1'
-- 'another_dir/some_file_2'
-- 'another_dir/some_file_3'
+- `some_dir/some_file_1`
+- `some_dir/some_file_2`
+- `some_dir/some_file_3`
+- `another_dir/some_file_1`
+- `another_dir/some_file_2`
+- `another_dir/some_file_3`
 
-Query the number of rows in these files:
+Query the total number of rows in all files:
 
 ``` sql
 SELECT count(*) FROM file('{some,another}_dir/some_file_{1..3}', 'TSV', 'name String, value UInt32');
 ```
 
-Query the number of rows in all files of these two directories:
+An alternative path expression which achieves the same:
 
 ``` sql
 SELECT count(*) FROM file('{some,another}_dir/*', 'TSV', 'name String, value UInt32');
@@ -176,7 +169,7 @@ If your listing of files contains number ranges with leading zeros, use the cons
 
 **Example**
 
-Query the data from files named `file000`, `file001`, … , `file999`:
+Query the total number of rows in files named `file000`, `file001`, … , `file999`:
 
 ``` sql
 SELECT count(*) FROM file('big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String, value UInt32');
@@ -184,7 +177,7 @@ SELECT count(*) FROM file('big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String,
 
 **Example**
 
-Query the data from all files inside `big_dir` directory recursively:
+Query the total number of rows from all files inside directory `big_dir/` recursively:
 
 ``` sql
 SELECT count(*) FROM file('big_dir/**', 'CSV', 'name String, value UInt32');
@@ -192,7 +185,7 @@ SELECT count(*) FROM file('big_dir/**', 'CSV', 'name String, value UInt32');
 
 **Example**
 
-Query the data from all `file002` files from any folder inside `big_dir` directory recursively:
+Query the total number of rows from all files `file002` inside any folder in directory `big_dir/` recursively:
 
 ``` sql
 SELECT count(*) FROM file('big_dir/**/file002', 'CSV', 'name String, value UInt32');

docs/en/sql-reference/table-functions/remote.md

@@ -6,7 +6,7 @@ sidebar_label: remote
 
 # remote, remoteSecure
 
-Allows accessing remote servers, including migration of data, without creating a [Distributed](../../engines/table-engines/special/distributed.md) table. `remoteSecure` - same as `remote` but with a secured connection.
+Table function `remote` allows to access remote servers on-the-fly, i.e. without creating a [Distributed](../../engines/table-engines/special/distributed.md) table. Table function `remoteSecure` is same as `remote` but over a secure connection.
 
 Both functions can be used in `SELECT` and `INSERT` queries.
 
@@ -21,36 +21,36 @@ remoteSecure('addresses_expr', [db.table, 'user'[, 'password'], sharding_key])
 
 ## Parameters
 
-- `addresses_expr` — An expression that generates addresses of remote servers. This may be just one server address. The server address is `host:port`, or just `host`.
+- `addresses_expr` — A remote server address or an expression that generates multiple addresses of remote servers. Format: `host` or `host:port`.
 
-    The host can be specified as the server name, or as the IPv4 or IPv6 address. An IPv6 address is specified in square brackets.
+    The `host` can be specified as a server name, or as a IPv4 or IPv6 address. An IPv6 address must be specified in square brackets.
 
-    The port is the TCP port on the remote server. If the port is omitted, it uses [tcp_port](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port) from the server’s config file in `remote` (by default, 9000) and [tcp_port_secure](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port_secure) in `remoteSecure` (by default, 9440).
+    The `port` is the TCP port on the remote server. If the port is omitted, it uses [tcp_port](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port) from the server config file for table function `remote` (by default, 9000) and [tcp_port_secure](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port_secure) for table function `remoteSecure` (by default, 9440).
 
-    The port is required for an IPv6 address.
+    For IPv6 addresses, a port is required.
 
-    If only specify this parameter, `db` and `table` will use `system.one` by default.
+    If only parameter `addresses_expr` is specified, `db` and `table` will use `system.one` by default.
 
     Type: [String](../../sql-reference/data-types/string.md).
 
 - `db` — Database name. Type: [String](../../sql-reference/data-types/string.md).
 - `table` — Table name. Type: [String](../../sql-reference/data-types/string.md).
-- `user` — User name. If the user is not specified, `default` is used. Type: [String](../../sql-reference/data-types/string.md).
-- `password` — User password. If the password is not specified, an empty password is used. Type: [String](../../sql-reference/data-types/string.md).
+- `user` — User name. If not specified, `default` is used. Type: [String](../../sql-reference/data-types/string.md).
+- `password` — User password. If not specified, an empty password is used. Type: [String](../../sql-reference/data-types/string.md).
 - `sharding_key` — Sharding key to support distributing data across nodes. For example: `insert into remote('127.0.0.1:9000,127.0.0.2', db, table, 'default', rand())`. Type: [UInt32](../../sql-reference/data-types/int-uint.md).
 
 ## Returned value
 
-The dataset from remote servers.
+A table located on a remote server.
 
 ## Usage
 
-Unless you are migrating data from one system to another, using the `remote` table function is less optimal than creating a `Distributed` table because in this case the server connection is re-established for every request. Also, if hostnames are set, the names are resolved, and errors are not counted when working with various replicas. When processing a large number of queries, always create the `Distributed` table ahead of time, and do not use the `remote` table function.
+As table functions `remote` and `remoteSecure` re-establish the connection for each request, it is recommended to use a `Distributed` table instead. Also, if hostnames are set, the names are resolved, and errors are not counted when working with various replicas. When processing a large number of queries, always create the `Distributed` table ahead of time, and do not use the `remote` table function.
 
 The `remote` table function can be useful in the following cases:
 
-- Migrating data from one system to another
-- Accessing a specific server for data comparison, debugging, and testing.
+- One-time data migration from one system to another
+- Accessing a specific server for data comparison, debugging, and testing, i.e. ad-hoc connections.
 - Queries between various ClickHouse clusters for research purposes.
 - Infrequent distributed requests that are made manually.
 - Distributed requests where the set of servers is re-defined each time.
@@ -68,7 +68,7 @@ localhost
 [2a02:6b8:0:1111::11]:9000
 ```
 
-Multiple addresses can be comma-separated. In this case, ClickHouse will use distributed processing, so it will send the query to all specified addresses (like shards with different data). Example:
+Multiple addresses can be comma-separated. In this case, ClickHouse will use distributed processing and send the query to all specified addresses (like shards with different data). Example:
 
 ``` text
 example01-01-1,example01-02-1
@@ -91,10 +91,13 @@ SELECT * FROM remote_table;
 ```
 
 ### Migration of tables from one system to another:
+
 This example uses one table from a sample dataset.  The database is `imdb`, and the table is `actors`.
 
 #### On the source ClickHouse system (the system that currently hosts the data)
+
 - Verify the source database and table name (`imdb.actors`)
+
   ```sql
   show databases
   ```
@@ -104,6 +107,7 @@ This example uses one table from a sample dataset.  The database is `imdb`, and
   ```
 
 - Get the CREATE TABLE statement from the source:
+
   ```
   select create_table_query
   from system.tables
@@ -111,6 +115,7 @@ This example uses one table from a sample dataset.  The database is `imdb`, and
   ```
 
   Response
+
   ```sql
   CREATE TABLE imdb.actors (`id` UInt32,
                             `first_name` String,
@@ -123,11 +128,13 @@ This example uses one table from a sample dataset.  The database is `imdb`, and
 #### On the destination ClickHouse system:
 
 - Create the destination database:
+
   ```sql
   CREATE DATABASE imdb
   ```
 
 - Using the CREATE TABLE statement from the source, create the destination:
+
   ```sql
   CREATE TABLE imdb.actors (`id` UInt32,
                             `first_name` String,
@@ -140,21 +147,23 @@ This example uses one table from a sample dataset.  The database is `imdb`, and
 #### Back on the source deployment:
 
 Insert into the new database and table created on the remote system.  You will need the host, port, username, password, destination database, and destination table.
+
 ```sql
 INSERT INTO FUNCTION
 remoteSecure('remote.clickhouse.cloud:9440', 'imdb.actors', 'USER', 'PASSWORD')
 SELECT * from imdb.actors
 ```
 
-## Globs in Addresses {#globs-in-addresses}
+## Globbing {#globs-in-addresses}
 
 Patterns in curly brackets `{ }` are used to generate a set of shards and to specify replicas. If there are multiple pairs of curly brackets, then the direct product of the corresponding sets is generated.
+
 The following pattern types are supported.
 
-- {*a*,*b*} - Any number of variants separated by a comma. The pattern is replaced with *a* in the first shard address and it is replaced with *b* in the second shard address and so on. For instance, `example0{1,2}-1` generates addresses `example01-1` and `example02-1`.
-- {*n*..*m*} - A range of numbers. This pattern generates shard addresses with incrementing indices from *n* to *m*. `example0{1..2}-1` generates `example01-1` and `example02-1`.
-- {*0n*..*0m*} - A range of numbers with leading zeroes. This modification preserves leading zeroes in indices. The pattern `example{01..03}-1` generates `example01-1`, `example02-1` and `example03-1`.
-- {*a*|*b*} - Any number of variants separated by a `|`. The pattern specifies replicas. For instance, `example01-{1|2}` generates replicas `example01-1` and `example01-2`.
+- `{a,b,c}` - Represents any of alternative strings `a`, `b` or `c`. The pattern is replaced with `a` in the first shard address and replaced with `b` in the second shard address and so on. For instance, `example0{1,2}-1` generates addresses `example01-1` and `example02-1`.
+- `{N..M}` - A range of numbers. This pattern generates shard addresses with incrementing indices from `N` to (and including) `M`. For instance, `example0{1..2}-1` generates `example01-1` and `example02-1`.
+- `{0n..0m}` - A range of numbers with leading zeroes. This pattern preserves leading zeroes in indices. For instance, `example{01..03}-1` generates `example01-1`, `example02-1` and `example03-1`.
+- `{a|b}` - Any number of variants separated by a `|`. The pattern specifies replicas. For instance, `example01-{1|2}` generates replicas `example01-1` and `example01-2`.
 
 The query will be sent to the first healthy replica. However, for `remote` the replicas are iterated in the order currently set in the [load_balancing](../../operations/settings/settings.md#settings-load_balancing) setting.
 The number of generated addresses is limited by [table_function_remote_max_addresses](../../operations/settings/settings.md#table_function_remote_max_addresses) setting.

docs/en/sql-reference/window-functions/index.md

@@ -86,14 +86,14 @@ WINDOW window_name as ([[PARTITION BY grouping_column] [ORDER BY sorting_column]
 
 These functions can be used only as a window function.
 
-`row_number()` -	Number the current row within its partition starting from 1.
-`first_value(x)` -	Return the first non-NULL value evaluated within its ordered frame.
-`last_value(x)` -	Return the last non-NULL value evaluated within its ordered frame.
-`nth_value(x, offset)` - Return the first non-NULL value evaluated against the nth row (offset) in its ordered frame.
-`rank()` -	Rank the current row within its partition with gaps.
-`dense_rank()`	- Rank the current row within its partition without gaps.
-`lagInFrame(x)` - Return a value evaluated at the row that is at a specified physical offset row before the current row within the ordered frame.
-`leadInFrame(x)` - Return a value evaluated at the row that is offset rows after the current row within the ordered frame.
+- `row_number()` - Number the current row within its partition starting from 1.
+- `first_value(x)` - Return the first non-NULL value evaluated within its ordered frame.
+- `last_value(x)` -	Return the last non-NULL value evaluated within its ordered frame.
+- `nth_value(x, offset)` - Return the first non-NULL value evaluated against the nth row (offset) in its ordered frame.
+- `rank()` - Rank the current row within its partition with gaps.
+- `dense_rank()` - Rank the current row within its partition without gaps.
+- `lagInFrame(x)` - Return a value evaluated at the row that is at a specified physical offset row before the current row within the ordered frame.
+- `leadInFrame(x)` - Return a value evaluated at the row that is offset rows after the current row within the ordered frame.
 
 ```text
       PARTITION

docs/ru/operations/server-configuration-parameters/settings.md

@@ -277,8 +277,10 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
 
 Если `true`, то каждый словарь создаётся при первом использовании. Если словарь не удалось создать, то вызов функции, использующей словарь, сгенерирует исключение.
 
-Если `false`, то все словари создаются при старте сервера, если словарь или словари создаются слишком долго или создаются с ошибкой, то сервер загружается без
-этих словарей и продолжает попытки создать эти словари.
+Если `false`, сервер начнет загрузку всех словарей на старте сервера.
+Словари загружаются в фоне. Сервер не ждет на старте, пока словари закончат загружаться
+(исключение: если `wait_dictionaries_load_at_startup` установлена в `true` - см. ниже).
+Когда словарь используется в запросе первый раз, этот запрос будет ждать окончания загрузки словаря, если он еще не загрузился.
 
 По умолчанию - `true`.
 
@@ -1718,6 +1720,24 @@ TCP порт для защищённого обмена данными с кли
 <users_config>users.xml</users_config>
 ```
 
+## wait_dictionaries_load_at_startup {#wait_dictionaries_load_at_startup}
+
+Если `false`, то сервер не будет ждать на старте, пока словари закончат загружаться.
+Это позволяет ClickHouse стартовать быстрее.
+
+Если `true`, то ClickHouse будет ждать на старте до окончания загрузки всех словарей (успешно или нет)
+перед тем, как начать принимать соединения.
+Это может привести к медленному старту ClickHouse, однако после этого некоторые запросы могут выполняться быстрее
+(потому что им не придется ждать окончания загрузки используемых словарей).
+
+По умолчанию - `false`.
+
+**Пример**
+
+``` xml
+<wait_dictionaries_load_at_startup>false</wait_dictionaries_load_at_startup>
+```
+
 ## zookeeper {#server-settings_zookeeper}
 
 Содержит параметры, позволяющие ClickHouse взаимодействовать с кластером [ZooKeeper](http://zookeeper.apache.org/).

docs/ru/operations/utilities/backupview.md

@@ -0,0 +1,50 @@
+---
+slug: /en/operations/utilities/backupview
+title: clickhouse_backupview
+---
+
+# clickhouse_backupview {#clickhouse_backupview}
+
+Модуль на Питоне для анализа бэкапов, созданных командой [BACKUP](https://clickhouse.com/docs/ru/operations/backup)
+Главная идея этого модуля была в том, чтобы позволить извлечение информации из бэкапа без выполнения команды RESTORE.
+
+Этот модуль содержит функции для
+- получения списка файлов внутри бэкапа
+- чтения файлов из бэкапа
+- получения информации в читаемом виде о базах данных, таблицах, партах, содержащихся в бэкапе
+- проверки целостности бэкапа
+
+## Пример:
+
+```python
+from clickhouse_backupview import open_backup, S3, FileInfo
+
+# Открыть бэкап. Можно также использовать локальный путь:
+# backup = open_backup("/backups/my_backup_1/")
+backup = open_backup(S3("uri", "access_key_id", "secret_access_key"))
+
+# Получить список баз данных внутри бэкапа.
+print(backup.get_databases()))
+
+# Получить список таблиц внутри бэкапа,
+# и для каждой таблицы получить ее определение а также список партов и партиций.
+for db in backup.get_databases():
+    for tbl in backup.get_tables(database=db):
+        print(backup.get_create_query(database=db, table=tbl))
+        print(backup.get_partitions(database=db, table=tbl))
+        print(backup.get_parts(database=db, table=tbl))
+
+# Извлечь все содержимое бэкапа.
+backup.extract_all(table="mydb.mytable", out='/tmp/my_backup_1/all/')
+
+# Извлечь данные конкретной таблицы.
+backup.extract_table_data(table="mydb.mytable", out='/tmp/my_backup_1/mytable/')
+
+# Извлечь одну партицию из бэкапа.
+backup.extract_table_data(table="mydb.mytable", partition="202201", out='/tmp/my_backup_1/202201/')
+
+# Извлечь один парт из бэкапа.
+backup.extract_table_data(table="mydb.mytable", part="202201_100_200_3", out='/tmp/my_backup_1/202201_100_200_3/')
+```
+
+Больше примеров смотрите в [тесте](https://github.com/ClickHouse/ClickHouse/blob/master/utils/backupview/test/test.py).

docs/ru/operations/utilities/index.md

@@ -13,3 +13,4 @@ sidebar_position: 56
 -   [ClickHouse obfuscator](../../operations/utilities/clickhouse-obfuscator.md) — обфусцирует данные.
 -   [ClickHouse compressor](../../operations/utilities/clickhouse-compressor.md) — упаковывает и распаковывает данные.
 -   [clickhouse-odbc-bridge](../../operations/utilities/odbc-bridge.md) — прокси-сервер для ODBC.
+-   [clickhouse_backupview](../../operations/utilities/backupview.md) — модуль на Питоне для анализа бэкапов ClickHouse.

programs/CMakeLists.txt

@@ -432,6 +432,11 @@ if (USE_BINARY_HASH)
     add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .clickhouse.hash=hash clickhouse COMMENT "Adding section '.clickhouse.hash' to clickhouse binary" VERBATIM)
 endif()
 
+if (CHECK_LARGE_OBJECT_SIZES)
+    add_custom_command(TARGET clickhouse POST_BUILD
+        COMMAND "${CMAKE_SOURCE_DIR}/utils/check-style/check-large-objects.sh" "${CMAKE_BINARY_DIR}")
+endif ()
+
 if (SPLIT_DEBUG_SYMBOLS)
     clickhouse_split_debug_symbols(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH clickhouse)
 else()

programs/client/Client.cpp

@@ -17,18 +17,15 @@
 #include "Core/Protocol.h"
 #include "Parsers/formatAST.h"
 
-#include <base/find_symbols.h>
-
 #include <Access/AccessControl.h>
 
-#include "config_version.h"
+#include <Common/config_version.h>
 #include <Common/Exception.h>
 #include <Common/formatReadable.h>
 #include <Common/TerminalSize.h>
 #include <Common/Config/ConfigProcessor.h>
 #include <Common/Config/getClientConfigPath.h>
 
-#include <Core/QueryProcessingStage.h>
 #include <Columns/ColumnString.h>
 #include <Poco/Util/Application.h>
 

programs/keeper/Keeper.cpp

@@ -35,7 +35,7 @@
 
 #include "Core/Defines.h"
 #include "config.h"
-#include "config_version.h"
+#include <Common/config_version.h>
 #include "config_tools.h"
 
 

programs/local/LocalServer.cpp

@@ -763,7 +763,7 @@ void LocalServer::processConfig()
         {
             DatabaseCatalog::instance().createBackgroundTasks();
             loadMetadata(global_context);
-            DatabaseCatalog::instance().startupBackgroundCleanup();
+            DatabaseCatalog::instance().startupBackgroundTasks();
         }
 
         /// For ClickHouse local if path is not set the loader will be disabled.

programs/server/Server.cpp

@@ -98,7 +98,7 @@
 #include <unordered_set>
 
 #include "config.h"
-#include "config_version.h"
+#include <Common/config_version.h>
 
 #if defined(OS_LINUX)
 #    include <cstdlib>
@@ -1372,6 +1372,8 @@ try
 
                 global_context->reloadAuxiliaryZooKeepersConfigIfChanged(config);
 
+                global_context->reloadQueryMaskingRulesIfChanged(config);
+
                 std::lock_guard lock(servers_lock);
                 updateServers(*config, server_pool, async_metrics, servers, servers_to_start_before_tables);
             }
@@ -1691,7 +1693,7 @@ try
         /// Then, load remaining databases
         loadMetadata(global_context, default_database);
         convertDatabasesEnginesIfNeed(global_context);
-        database_catalog.startupBackgroundCleanup();
+        database_catalog.startupBackgroundTasks();
         /// After loading validate that default database exists
         database_catalog.assertDatabaseExists(default_database);
         /// Load user-defined SQL functions.
@@ -1816,6 +1818,9 @@ try
         try
         {
             global_context->loadOrReloadDictionaries(config());
+
+            if (config().getBool("wait_dictionaries_load_at_startup", false))
+                global_context->waitForDictionariesLoad();
         }
         catch (...)
         {

programs/server/config.xml

@@ -1266,6 +1266,17 @@
     -->
     <dictionaries_config>*_dictionary.*ml</dictionaries_config>
 
+    <!-- Load dictionaries lazily, i.e. a dictionary will be loaded when it's used for the first time.
+         "false" means ClickHouse will start loading dictionaries immediately at startup.
+    -->
+    <dictionaries_lazy_load>true</dictionaries_lazy_load>
+
+    <!-- Wait at startup until all the dictionaries finish their loading (successfully or not)
+         before listening to connections. Setting this to 1 can make ClickHouse start slowly,
+         however some queries can be executed faster (because it won't have to wait for the used dictionaries to be load).
+    -->
+    <wait_dictionaries_load_at_startup>false</wait_dictionaries_load_at_startup>
+
     <!-- Configuration of user defined executable functions -->
     <user_defined_executable_functions_config>*_function.*ml</user_defined_executable_functions_config>
 

src/Analyzer/IQueryTreeNode.h

@@ -182,7 +182,7 @@ public:
 
     struct ConvertToASTOptions
     {
-        /// Add _CAST if constant litral type is different from column type
+        /// Add _CAST if constant literal type is different from column type
         bool add_cast_for_constants = true;
 
         /// Identifiers are fully qualified (`database.table.column`), otherwise names are just column names (`column`)

src/Analyzer/InDepthQueryTreeVisitor.h

@@ -188,7 +188,7 @@ private:
         if (auto * table_function_node = parent->as<TableFunctionNode>())
         {
             if (child != table_function_node->getArgumentsNode())
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "TableFunctioNode is expected to have only one child node");
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "TableFunctionNode is expected to have only one child node");
 
             const auto & unresolved_indexes = table_function_node->getUnresolvedArgumentIndexes();
 

src/Analyzer/Passes/QueryAnalysisPass.cpp

@@ -1952,7 +1952,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden
         subquery_context->setSetting("use_structure_from_insertion_table_in_table_functions", false);
 
         auto options = SelectQueryOptions(QueryProcessingStage::Complete, scope.subquery_depth, true /*is_subquery*/);
-        auto interpreter = std::make_unique<InterpreterSelectQueryAnalyzer>(node->toAST(), subquery_context, options);
+        auto interpreter = std::make_unique<InterpreterSelectQueryAnalyzer>(node->toAST(), subquery_context, subquery_context->getViewSource(), options);
 
         auto io = interpreter->execute();
 
@@ -4896,6 +4896,25 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
         }
         else
         {
+            /// Replace storage with values storage of insertion block
+            if (StoragePtr storage = scope.context->getViewSource())
+            {
+                if (auto * query_node = in_second_argument->as<QueryNode>())
+                {
+                    auto table_expression = extractLeftTableExpression(query_node->getJoinTree());
+                    if (auto * query_table_node = table_expression->as<TableNode>())
+                    {
+                        if (query_table_node->getStorageID().getFullNameNotQuoted() == storage->getStorageID().getFullNameNotQuoted())
+                        {
+                            auto replacement_table_expression = std::make_shared<TableNode>(storage, scope.context);
+                            if (std::optional<TableExpressionModifiers> table_expression_modifiers = query_table_node->getTableExpressionModifiers())
+                                replacement_table_expression->setTableExpressionModifiers(*table_expression_modifiers);
+                            in_second_argument = in_second_argument->cloneAndReplace(table_expression, std::move(replacement_table_expression));
+                        }
+                    }
+                }
+            }
+
             resolveExpressionNode(in_second_argument, scope, false /*allow_lambda_expression*/, true /*allow_table_expression*/);
         }
     }
@@ -6327,6 +6346,94 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node,
                 table_function_name);
     }
 
+    QueryTreeNodes result_table_function_arguments;
+
+    auto skip_analysis_arguments_indexes = table_function_ptr->skipAnalysisForArguments(table_function_node, scope_context);
+
+    auto & table_function_arguments = table_function_node_typed.getArguments().getNodes();
+    size_t table_function_arguments_size = table_function_arguments.size();
+
+    for (size_t table_function_argument_index = 0; table_function_argument_index < table_function_arguments_size; ++table_function_argument_index)
+    {
+        auto & table_function_argument = table_function_arguments[table_function_argument_index];
+
+        auto skip_argument_index_it = std::find(skip_analysis_arguments_indexes.begin(),
+            skip_analysis_arguments_indexes.end(),
+            table_function_argument_index);
+        if (skip_argument_index_it != skip_analysis_arguments_indexes.end())
+        {
+            result_table_function_arguments.push_back(table_function_argument);
+            continue;
+        }
+
+        if (auto * identifier_node = table_function_argument->as<IdentifierNode>())
+        {
+            const auto & unresolved_identifier = identifier_node->getIdentifier();
+            auto identifier_resolve_result = tryResolveIdentifier({unresolved_identifier, IdentifierLookupContext::EXPRESSION}, scope);
+            auto resolved_identifier = std::move(identifier_resolve_result.resolved_identifier);
+
+            if (resolved_identifier && resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
+                result_table_function_arguments.push_back(std::move(resolved_identifier));
+            else
+                result_table_function_arguments.push_back(table_function_argument);
+
+            continue;
+        }
+        else if (auto * table_function_argument_function = table_function_argument->as<FunctionNode>())
+        {
+            const auto & table_function_argument_function_name = table_function_argument_function->getFunctionName();
+            if (TableFunctionFactory::instance().isTableFunctionName(table_function_argument_function_name))
+            {
+                auto table_function_node_to_resolve_typed = std::make_shared<TableFunctionNode>(table_function_argument_function_name);
+                table_function_node_to_resolve_typed->getArgumentsNode() = table_function_argument_function->getArgumentsNode();
+
+                QueryTreeNodePtr table_function_node_to_resolve = std::move(table_function_node_to_resolve_typed);
+                resolveTableFunction(table_function_node_to_resolve, scope, expressions_visitor, true /*nested_table_function*/);
+
+                result_table_function_arguments.push_back(std::move(table_function_node_to_resolve));
+                continue;
+            }
+        }
+
+        /** Table functions arguments can contain expressions with invalid identifiers.
+          * We cannot skip analysis for such arguments, because some table functions cannot provide
+          * information if analysis for argument should be skipped until other arguments will be resolved.
+          *
+          * Example: SELECT key from remote('127.0.0.{1,2}', view(select number AS key from numbers(2)), cityHash64(key));
+          * Example: SELECT id from remote('127.0.0.{1,2}', 'default', 'test_table', cityHash64(id));
+          */
+        try
+        {
+            resolveExpressionNode(table_function_argument, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
+        }
+        catch (const Exception & exception)
+        {
+            if (exception.code() == ErrorCodes::UNKNOWN_IDENTIFIER)
+            {
+                result_table_function_arguments.push_back(table_function_argument);
+                continue;
+            }
+
+            throw;
+        }
+
+        if (auto * expression_list = table_function_argument->as<ListNode>())
+        {
+            for (auto & expression_list_node : expression_list->getNodes())
+                result_table_function_arguments.push_back(expression_list_node);
+        }
+        else
+        {
+            result_table_function_arguments.push_back(table_function_argument);
+        }
+    }
+
+    table_function_node_typed.getArguments().getNodes() = std::move(result_table_function_arguments);
+
+    auto table_function_ast = table_function_node_typed.toAST();
+    table_function_ptr->parseArguments(table_function_ast, scope_context);
+
+
     uint64_t use_structure_from_insertion_table_in_table_functions = scope_context->getSettingsRef().use_structure_from_insertion_table_in_table_functions;
     if (!nested_table_function &&
         use_structure_from_insertion_table_in_table_functions &&
@@ -6468,93 +6575,6 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node,
         }
     }
 
-    QueryTreeNodes result_table_function_arguments;
-
-    auto skip_analysis_arguments_indexes = table_function_ptr->skipAnalysisForArguments(table_function_node, scope_context);
-
-    auto & table_function_arguments = table_function_node_typed.getArguments().getNodes();
-    size_t table_function_arguments_size = table_function_arguments.size();
-
-    for (size_t table_function_argument_index = 0; table_function_argument_index < table_function_arguments_size; ++table_function_argument_index)
-    {
-        auto & table_function_argument = table_function_arguments[table_function_argument_index];
-
-        auto skip_argument_index_it = std::find(skip_analysis_arguments_indexes.begin(),
-            skip_analysis_arguments_indexes.end(),
-            table_function_argument_index);
-        if (skip_argument_index_it != skip_analysis_arguments_indexes.end())
-        {
-            result_table_function_arguments.push_back(table_function_argument);
-            continue;
-        }
-
-        if (auto * identifier_node = table_function_argument->as<IdentifierNode>())
-        {
-            const auto & unresolved_identifier = identifier_node->getIdentifier();
-            auto identifier_resolve_result = tryResolveIdentifier({unresolved_identifier, IdentifierLookupContext::EXPRESSION}, scope);
-            auto resolved_identifier = std::move(identifier_resolve_result.resolved_identifier);
-
-            if (resolved_identifier && resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
-                result_table_function_arguments.push_back(std::move(resolved_identifier));
-            else
-                result_table_function_arguments.push_back(table_function_argument);
-
-            continue;
-        }
-        else if (auto * table_function_argument_function = table_function_argument->as<FunctionNode>())
-        {
-            const auto & table_function_argument_function_name = table_function_argument_function->getFunctionName();
-            if (TableFunctionFactory::instance().isTableFunctionName(table_function_argument_function_name))
-            {
-                auto table_function_node_to_resolve_typed = std::make_shared<TableFunctionNode>(table_function_argument_function_name);
-                table_function_node_to_resolve_typed->getArgumentsNode() = table_function_argument_function->getArgumentsNode();
-
-                QueryTreeNodePtr table_function_node_to_resolve = std::move(table_function_node_to_resolve_typed);
-                resolveTableFunction(table_function_node_to_resolve, scope, expressions_visitor, true /*nested_table_function*/);
-
-                result_table_function_arguments.push_back(std::move(table_function_node_to_resolve));
-                continue;
-            }
-        }
-
-        /** Table functions arguments can contain expressions with invalid identifiers.
-          * We cannot skip analysis for such arguments, because some table functions cannot provide
-          * information if analysis for argument should be skipped until other arguments will be resolved.
-          *
-          * Example: SELECT key from remote('127.0.0.{1,2}', view(select number AS key from numbers(2)), cityHash64(key));
-          * Example: SELECT id from remote('127.0.0.{1,2}', 'default', 'test_table', cityHash64(id));
-          */
-        try
-        {
-            resolveExpressionNode(table_function_argument, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
-        }
-        catch (const Exception & exception)
-        {
-            if (exception.code() == ErrorCodes::UNKNOWN_IDENTIFIER)
-            {
-                result_table_function_arguments.push_back(table_function_argument);
-                continue;
-            }
-
-            throw;
-        }
-
-        if (auto * expression_list = table_function_argument->as<ListNode>())
-        {
-            for (auto & expression_list_node : expression_list->getNodes())
-                result_table_function_arguments.push_back(expression_list_node);
-        }
-        else
-        {
-            result_table_function_arguments.push_back(table_function_argument);
-        }
-    }
-
-    table_function_node_typed.getArguments().getNodes() = std::move(result_table_function_arguments);
-
-    auto table_function_ast = table_function_node_typed.toAST();
-    table_function_ptr->parseArguments(table_function_ast, scope_context);
-
     auto table_function_storage = scope_context->getQueryContext()->executeTableFunction(table_function_ast, table_function_ptr);
     table_function_node_typed.resolve(std::move(table_function_ptr), std::move(table_function_storage), scope_context, std::move(skip_analysis_arguments_indexes));
 }

src/Analyzer/Passes/RemoveUnusedProjectionColumnsPass.cpp

@@ -0,0 +1,168 @@
+#include <Analyzer/Passes/RemoveUnusedProjectionColumnsPass.h>
+
+#include <Functions/FunctionFactory.h>
+
+#include <Analyzer/InDepthQueryTreeVisitor.h>
+#include <Analyzer/FunctionNode.h>
+#include <Analyzer/QueryNode.h>
+#include <Analyzer/ColumnNode.h>
+#include <Analyzer/SortNode.h>
+#include <Analyzer/AggregationUtils.h>
+#include <Analyzer/Utils.h>
+
+namespace DB
+{
+
+namespace
+{
+
+class CollectUsedColumnsVisitor : public InDepthQueryTreeVisitorWithContext<CollectUsedColumnsVisitor>
+{
+public:
+    using Base = InDepthQueryTreeVisitorWithContext<CollectUsedColumnsVisitor>;
+    using Base::Base;
+
+    bool needChildVisit(QueryTreeNodePtr &, QueryTreeNodePtr & child)
+    {
+        if (isQueryOrUnionNode(child))
+        {
+            subqueries_nodes_to_visit.insert(child);
+            return false;
+        }
+
+        return true;
+    }
+
+    void enterImpl(QueryTreeNodePtr & node)
+    {
+        auto node_type = node->getNodeType();
+
+        if (node_type == QueryTreeNodeType::QUERY)
+        {
+            auto & query_node = node->as<QueryNode &>();
+            auto table_expressions = extractTableExpressions(query_node.getJoinTree());
+            for (const auto & table_expression : table_expressions)
+                if (isQueryOrUnionNode(table_expression))
+                    query_or_union_node_to_used_columns.emplace(table_expression, std::unordered_set<std::string>());
+
+            return;
+        }
+
+        if (node_type != QueryTreeNodeType::COLUMN)
+            return;
+
+        auto & column_node = node->as<ColumnNode &>();
+        auto column_source_node = column_node.getColumnSource();
+        auto column_source_node_type = column_source_node->getNodeType();
+
+        if (column_source_node_type == QueryTreeNodeType::QUERY || column_source_node_type == QueryTreeNodeType::UNION)
+            query_or_union_node_to_used_columns[column_source_node].insert(column_node.getColumnName());
+    }
+
+    void reset()
+    {
+        subqueries_nodes_to_visit.clear();
+        query_or_union_node_to_used_columns.clear();
+    }
+
+    std::unordered_set<QueryTreeNodePtr> subqueries_nodes_to_visit;
+    std::unordered_map<QueryTreeNodePtr, std::unordered_set<std::string>> query_or_union_node_to_used_columns;
+};
+
+std::unordered_set<size_t> convertUsedColumnNamesToUsedProjectionIndexes(const QueryTreeNodePtr & query_or_union_node, const std::unordered_set<std::string> & used_column_names)
+{
+    std::unordered_set<size_t> result;
+
+    auto * union_node = query_or_union_node->as<UnionNode>();
+    auto * query_node = query_or_union_node->as<QueryNode>();
+
+    const auto & projection_columns = query_node ? query_node->getProjectionColumns() : union_node->computeProjectionColumns();
+    size_t projection_columns_size = projection_columns.size();
+
+    for (size_t i = 0; i < projection_columns_size; ++i)
+    {
+        const auto & projection_column = projection_columns[i];
+        if (used_column_names.contains(projection_column.name))
+            result.insert(i);
+    }
+
+    return result;
+}
+
+/// We cannot remove aggregate functions, if query does not contain GROUP BY or arrayJoin from subquery projection
+void updateUsedProjectionIndexes(const QueryTreeNodePtr & query_or_union_node, std::unordered_set<size_t> & used_projection_columns_indexes)
+{
+    if (auto * union_node = query_or_union_node->as<UnionNode>())
+    {
+        auto union_node_mode = union_node->getUnionMode();
+        bool is_distinct = union_node_mode == SelectUnionMode::UNION_DISTINCT ||
+            union_node_mode == SelectUnionMode::INTERSECT_DISTINCT ||
+            union_node_mode == SelectUnionMode::EXCEPT_DISTINCT;
+
+        if (is_distinct)
+        {
+            auto union_projection_columns = union_node->computeProjectionColumns();
+            size_t union_projection_columns_size = union_projection_columns.size();
+
+            for (size_t i = 0; i < union_projection_columns_size; ++i)
+                used_projection_columns_indexes.insert(i);
+
+            return;
+        }
+
+        for (auto & query_node : union_node->getQueries().getNodes())
+            updateUsedProjectionIndexes(query_node, used_projection_columns_indexes);
+        return;
+    }
+
+    const auto & query_node = query_or_union_node->as<const QueryNode &>();
+    const auto & projection_nodes = query_node.getProjection().getNodes();
+    size_t projection_nodes_size = projection_nodes.size();
+
+    for (size_t i = 0; i < projection_nodes_size; ++i)
+    {
+        const auto & projection_node = projection_nodes[i];
+        if ((!query_node.hasGroupBy() && hasAggregateFunctionNodes(projection_node)) || hasFunctionNode(projection_node, "arrayJoin"))
+            used_projection_columns_indexes.insert(i);
+    }
+}
+
+}
+
+void RemoveUnusedProjectionColumnsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
+{
+    std::vector<QueryTreeNodePtr> nodes_to_visit;
+    nodes_to_visit.push_back(query_tree_node);
+
+    CollectUsedColumnsVisitor visitor(std::move(context));
+
+    while (!nodes_to_visit.empty())
+    {
+        auto node_to_visit = std::move(nodes_to_visit.back());
+        nodes_to_visit.pop_back();
+
+        visitor.visit(node_to_visit);
+
+        for (auto & [query_or_union_node, used_columns] : visitor.query_or_union_node_to_used_columns)
+        {
+            auto used_projection_indexes = convertUsedColumnNamesToUsedProjectionIndexes(query_or_union_node, used_columns);
+            updateUsedProjectionIndexes(query_or_union_node, used_projection_indexes);
+
+            /// Keep at least 1 column if used projection columns are empty
+            if (used_projection_indexes.empty())
+                used_projection_indexes.insert(0);
+
+            if (auto * union_node = query_or_union_node->as<UnionNode>())
+                union_node->removeUnusedProjectionColumns(used_projection_indexes);
+            else if (auto * query_node = query_or_union_node->as<QueryNode>())
+                query_node->removeUnusedProjectionColumns(used_projection_indexes);
+        }
+
+        for (const auto & subquery_node_to_visit : visitor.subqueries_nodes_to_visit)
+            nodes_to_visit.push_back(subquery_node_to_visit);
+
+        visitor.reset();
+    }
+}
+
+}

src/Analyzer/Passes/RemoveUnusedProjectionColumnsPass.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <Analyzer/IQueryTreePass.h>
+
+namespace DB
+{
+
+/** Remove unused projection columns in subqueries.
+  *
+  * Example: SELECT a FROM (SELECT a, b FROM test_table);
+  * Result: SELECT a FROM (SELECT a FROM test_table);
+  */
+class RemoveUnusedProjectionColumnsPass final : public IQueryTreePass
+{
+public:
+    String getName() override { return "RemoveUnusedProjectionColumnsPass"; }
+
+    String getDescription() override { return "Remove unused projection columns in subqueries."; }
+
+    void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
+
+};
+
+}

src/Analyzer/QueryNode.cpp

@@ -46,6 +46,54 @@ QueryNode::QueryNode(ContextMutablePtr context_)
     : QueryNode(std::move(context_), {} /*settings_changes*/)
 {}
 
+void QueryNode::resolveProjectionColumns(NamesAndTypes projection_columns_value)
+{
+    if (projection_columns_value.size() != getProjection().getNodes().size())
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected projection columns size to match projection nodes size");
+
+    projection_columns = std::move(projection_columns_value);
+}
+
+void QueryNode::removeUnusedProjectionColumns(const std::unordered_set<std::string> & used_projection_columns)
+{
+    auto & projection_nodes = getProjection().getNodes();
+    size_t projection_columns_size = projection_columns.size();
+    size_t write_index = 0;
+
+    for (size_t i = 0; i < projection_columns_size; ++i)
+    {
+        if (!used_projection_columns.contains(projection_columns[i].name))
+            continue;
+
+        projection_nodes[write_index] = projection_nodes[i];
+        projection_columns[write_index] = projection_columns[i];
+        ++write_index;
+    }
+
+    projection_nodes.erase(projection_nodes.begin() + write_index, projection_nodes.end());
+    projection_columns.erase(projection_columns.begin() + write_index, projection_columns.end());
+}
+
+void QueryNode::removeUnusedProjectionColumns(const std::unordered_set<size_t> & used_projection_columns_indexes)
+{
+    auto & projection_nodes = getProjection().getNodes();
+    size_t projection_columns_size = projection_columns.size();
+    size_t write_index = 0;
+
+    for (size_t i = 0; i < projection_columns_size; ++i)
+    {
+        if (!used_projection_columns_indexes.contains(i))
+            continue;
+
+        projection_nodes[write_index] = projection_nodes[i];
+        projection_columns[write_index] = projection_columns[i];
+        ++write_index;
+    }
+
+    projection_nodes.erase(projection_nodes.begin() + write_index, projection_nodes.end());
+    projection_columns.erase(projection_columns.begin() + write_index, projection_columns.end());
+}
+
 void QueryNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const
 {
     buffer << std::string(indent, ' ') << "QUERY id: " << format_state.getNodeId(this);

src/Analyzer/QueryNode.h

@@ -556,10 +556,13 @@ public:
     }
 
     /// Resolve query node projection columns
-    void resolveProjectionColumns(NamesAndTypes projection_columns_value)
-    {
-        projection_columns = std::move(projection_columns_value);
-    }
+    void resolveProjectionColumns(NamesAndTypes projection_columns_value);
+
+    /// Remove unused projection columns
+    void removeUnusedProjectionColumns(const std::unordered_set<std::string> & used_projection_columns);
+
+    /// Remove unused projection columns
+    void removeUnusedProjectionColumns(const std::unordered_set<size_t> & used_projection_columns_indexes);
 
     QueryTreeNodeType getNodeType() const override
     {

src/Analyzer/QueryTreePassManager.cpp

@@ -17,6 +17,7 @@
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Analyzer/Utils.h>
 #include <Analyzer/Passes/QueryAnalysisPass.h>
+#include <Analyzer/Passes/RemoveUnusedProjectionColumnsPass.h>
 #include <Analyzer/Passes/CountDistinctPass.h>
 #include <Analyzer/Passes/UniqToCountPass.h>
 #include <Analyzer/Passes/FunctionToSubcolumnsPass.h>
@@ -243,6 +244,7 @@ void QueryTreePassManager::dump(WriteBuffer & buffer, size_t up_to_pass_index)
 void addQueryTreePasses(QueryTreePassManager & manager)
 {
     manager.addPass(std::make_unique<QueryAnalysisPass>());
+    manager.addPass(std::make_unique<RemoveUnusedProjectionColumnsPass>());
     manager.addPass(std::make_unique<FunctionToSubcolumnsPass>());
 
     manager.addPass(std::make_unique<ConvertLogicalExpressionToCNFPass>());

src/Analyzer/UnionNode.cpp

@@ -88,6 +88,41 @@ NamesAndTypes UnionNode::computeProjectionColumns() const
     return result_columns;
 }
 
+void UnionNode::removeUnusedProjectionColumns(const std::unordered_set<std::string> & used_projection_columns)
+{
+    auto projection_columns = computeProjectionColumns();
+    size_t projection_columns_size = projection_columns.size();
+    std::unordered_set<size_t> used_projection_column_indexes;
+
+    for (size_t i = 0; i < projection_columns_size; ++i)
+    {
+        const auto & projection_column = projection_columns[i];
+        if (used_projection_columns.contains(projection_column.name))
+            used_projection_column_indexes.insert(i);
+    }
+
+    auto & query_nodes = getQueries().getNodes();
+    for (auto & query_node : query_nodes)
+    {
+        if (auto * query_node_typed = query_node->as<QueryNode>())
+            query_node_typed->removeUnusedProjectionColumns(used_projection_column_indexes);
+        else if (auto * union_node_typed = query_node->as<UnionNode>())
+            union_node_typed->removeUnusedProjectionColumns(used_projection_column_indexes);
+    }
+}
+
+void UnionNode::removeUnusedProjectionColumns(const std::unordered_set<size_t> & used_projection_columns_indexes)
+{
+    auto & query_nodes = getQueries().getNodes();
+    for (auto & query_node : query_nodes)
+    {
+        if (auto * query_node_typed = query_node->as<QueryNode>())
+            query_node_typed->removeUnusedProjectionColumns(used_projection_columns_indexes);
+        else if (auto * union_node_typed = query_node->as<UnionNode>())
+            union_node_typed->removeUnusedProjectionColumns(used_projection_columns_indexes);
+    }
+}
+
 void UnionNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const
 {
     buffer << std::string(indent, ' ') << "UNION id: " << format_state.getNodeId(this);

src/Analyzer/UnionNode.h

@@ -129,6 +129,12 @@ public:
     /// Compute union node projection columns
     NamesAndTypes computeProjectionColumns() const;
 
+    /// Remove unused projection columns
+    void removeUnusedProjectionColumns(const std::unordered_set<std::string> & used_projection_columns);
+
+    /// Remove unused projection columns
+    void removeUnusedProjectionColumns(const std::unordered_set<size_t> & used_projection_columns_indexes);
+
     QueryTreeNodeType getNodeType() const override
     {
         return QueryTreeNodeType::UNION;

src/Analyzer/Utils.cpp

@@ -152,6 +152,17 @@ void makeUniqueColumnNamesInBlock(Block & block)
     }
 }
 
+bool isQueryOrUnionNode(const IQueryTreeNode * node)
+{
+    auto node_type = node->getNodeType();
+    return node_type == QueryTreeNodeType::QUERY || node_type == QueryTreeNodeType::UNION;
+}
+
+bool isQueryOrUnionNode(const QueryTreeNodePtr & node)
+{
+    return isQueryOrUnionNode(node.get());
+}
+
 QueryTreeNodePtr buildCastFunction(const QueryTreeNodePtr & expression,
     const DataTypePtr & type,
     const ContextPtr & context,

src/Analyzer/Utils.h

@@ -27,6 +27,12 @@ std::string getGlobalInFunctionNameForLocalInFunctionName(const std::string & fu
 /// Add unique suffix to names of duplicate columns in block
 void makeUniqueColumnNamesInBlock(Block & block);
 
+/// Returns true, if node has type QUERY or UNION
+bool isQueryOrUnionNode(const IQueryTreeNode * node);
+
+/// Returns true, if node has type QUERY or UNION
+bool isQueryOrUnionNode(const QueryTreeNodePtr & node);
+
 /** Build cast function that cast expression into type.
   * If resolve = true, then result cast function is resolved during build, otherwise
   * result cast function is not resolved during build.

src/CMakeLists.txt

@@ -18,7 +18,6 @@ include (../cmake/version.cmake)
 message (STATUS "Will build ${VERSION_FULL} revision ${VERSION_REVISION} ${VERSION_OFFICIAL}")
 include (configure_config.cmake)
 configure_file (Common/config.h.in ${CONFIG_INCLUDE_PATH}/config.h)
-configure_file (Common/config_version.h.in ${CONFIG_INCLUDE_PATH}/config_version.h)
 configure_file (Common/config_version.cpp.in ${CONFIG_INCLUDE_PATH}/config_version.cpp)
 
 if (USE_DEBUG_HELPERS)

src/Client/ClientBase.cpp

@@ -77,7 +77,7 @@
 #include <memory>
 #include <unordered_map>
 
-#include "config_version.h"
+#include <Common/config_version.h>
 #include "config.h"
 
 namespace fs = std::filesystem;
@@ -1797,7 +1797,12 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
     {
         const auto * logs_level_field = set_query->changes.tryGet(std::string_view{"send_logs_level"});
         if (logs_level_field)
-            updateLoggerLevel(logs_level_field->safeGet<String>());
+        {
+            auto logs_level = logs_level_field->safeGet<String>();
+            /// Check that setting value is correct before updating logger level.
+            SettingFieldLogsLevelTraits::fromString(logs_level);
+            updateLoggerLevel(logs_level);
+        }
     }
 
     if (const auto * create_user_query = parsed_query->as<ASTCreateUserQuery>())

src/Client/Connection.cpp

@@ -35,7 +35,7 @@
 #include <pcg_random.hpp>
 #include <base/scope_guard.h>
 
-#include "config_version.h"
+#include <Common/config_version.h>
 #include "config.h"
 
 #if USE_SSL
@@ -296,7 +296,7 @@ void Connection::sendHello()
                         "Parameters 'default_database', 'user' and 'password' must not contain ASCII control characters");
 
     writeVarUInt(Protocol::Client::Hello, *out);
-    writeStringBinary((VERSION_NAME " ") + client_name, *out);
+    writeStringBinary(std::string(VERSION_NAME) + " " + client_name, *out);
     writeVarUInt(VERSION_MAJOR, *out);
     writeVarUInt(VERSION_MINOR, *out);
     // NOTE For backward compatibility of the protocol, client cannot send its version_patch.

src/Client/LocalConnection.cpp

@@ -251,10 +251,12 @@ void LocalConnection::finishQuery()
     else if (state->pushing_async_executor)
     {
         state->pushing_async_executor->finish();
+        state->pushing_async_executor.reset();
     }
     else if (state->pushing_executor)
     {
         state->pushing_executor->finish();
+        state->pushing_executor.reset();
     }
 
     state->io.onFinish();

src/Client/QueryFuzzer.cpp

@@ -517,9 +517,9 @@ void QueryFuzzer::fuzzCreateQuery(ASTCreateQuery & create)
     SipHash sip_hash;
     sip_hash.update(original_name);
     if (create.columns_list)
-        create.columns_list->updateTreeHash(sip_hash);
+        create.columns_list->updateTreeHash(sip_hash, /*ignore_aliases=*/ true);
     if (create.storage)
-        create.storage->updateTreeHash(sip_hash);
+        create.storage->updateTreeHash(sip_hash, /*ignore_aliases=*/ true);
 
     const auto hash = getSipHash128AsPair(sip_hash);
 

src/Common/ClickHouseRevision.cpp

@@ -1,5 +1,5 @@
 #include <Common/ClickHouseRevision.h>
-#include "config_version.h"
+#include <Common/config_version.h>
 
 namespace ClickHouseRevision
 {

src/Common/Config/ConfigProcessor.cpp

@@ -330,6 +330,12 @@ void ConfigProcessor::mergeRecursive(XMLDocumentPtr config, Node * config_root,
                 {
                     Element & config_element = dynamic_cast<Element &>(*config_node);
 
+                    /// Remove substitution attributes from the merge target node if source node already has a value
+                    bool source_has_value = with_element.hasChildNodes();
+                    if (source_has_value)
+                        for (const auto & attr_name: SUBSTITUTION_ATTRS)
+                            config_element.removeAttribute(attr_name);
+
                     mergeAttributes(config_element, with_element);
                     mergeRecursive(config, config_node, with_node);
                 }
@@ -513,6 +519,9 @@ void ConfigProcessor::doIncludesRecursive(
 
     if (attr_nodes["from_zk"]) /// we have zookeeper subst
     {
+        if (node->hasChildNodes()) /// only allow substitution for nodes with no value
+            throw Poco::Exception("Element <" + node->nodeName() + "> has value, can't process from_zk substitution");
+
         contributing_zk_paths.insert(attr_nodes["from_zk"]->getNodeValue());
 
         if (zk_node_cache)
@@ -535,6 +544,9 @@ void ConfigProcessor::doIncludesRecursive(
 
     if (attr_nodes["from_env"]) /// we have env subst
     {
+        if (node->hasChildNodes()) /// only allow substitution for nodes with no value
+            throw Poco::Exception("Element <" + node->nodeName() + "> has value, can't process from_env substitution");
+
         XMLDocumentPtr env_document;
         auto get_env_node = [&](const std::string & name) -> const Node *
         {

src/Common/Dwarf.h

@@ -46,8 +46,8 @@ class Elf;
  * can parse Debug Information Entries (DIEs), abbreviations, attributes (of
  * all forms), and we can interpret bytecode for the line number VM.
  *
- * We can interpret DWARF records of version 2, 3, or 4, although we don't
- * actually support many of the version 4 features (such as VLIW, multiple
+ * We can interpret DWARF records of version 2, 3, 4, or 5, although we don't
+ * actually support many of the features of versions 4 and 5 (such as VLIW, multiple
  * operations per instruction)
  *
  * Note that the DWARF record parser does not allocate heap memory at all.

src/Common/ErrorCodes.cpp

@@ -583,7 +583,7 @@
     M(701, CLUSTER_DOESNT_EXIST) \
     M(702, CLIENT_INFO_DOES_NOT_MATCH) \
     M(703, INVALID_IDENTIFIER) \
-    M(704, CANNOT_USE_QUERY_CACHE_WITH_NONDETERMINISTIC_FUNCTIONS) \
+    M(704, QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS) \
     M(705, TABLE_NOT_EMPTY) \
     M(706, LIBSSH_ERROR) \
     M(999, KEEPER_EXCEPTION) \

src/Common/Exception.cpp

@@ -21,7 +21,7 @@
 #include <Common/LockMemoryExceptionInThread.h>
 #include <filesystem>
 
-#include "config_version.h"
+#include <Common/config_version.h>
 
 namespace fs = std::filesystem;
 

src/Common/Exception.h

@@ -73,8 +73,8 @@ protected:
     struct MessageMasked
     {
         std::string msg;
-        MessageMasked(const std::string & msg_);
-        MessageMasked(std::string && msg_);
+        explicit MessageMasked(const std::string & msg_);
+        explicit MessageMasked(std::string && msg_);
     };
 
     Exception(const MessageMasked & msg_masked, int code, bool remote_);
@@ -123,7 +123,7 @@ public:
     Exception(CreateFromSTDTag, const std::exception & exc);
 
     Exception * clone() const override { return new Exception(*this); }
-    void rethrow() const override { throw *this; }
+    void rethrow() const override { throw *this; } // NOLINT
     const char * name() const noexcept override { return "DB::Exception"; }
     const char * what() const noexcept override { return message().data(); }
 
@@ -181,7 +181,7 @@ public:
         : Exception(msg, code), saved_errno(saved_errno_), path(path_) {}
 
     ErrnoException * clone() const override { return new ErrnoException(*this); }
-    void rethrow() const override { throw *this; }
+    void rethrow() const override { throw *this; } // NOLINT
 
     int getErrno() const { return saved_errno; }
     std::optional<std::string> getPath() const { return path; }
@@ -219,7 +219,7 @@ public:
     void setFileName(const String & file_name_) { file_name = file_name_; }
 
     Exception * clone() const override { return new ParsingException(*this); }
-    void rethrow() const override { throw *this; }
+    void rethrow() const override { throw *this; } // NOLINT
 
 private:
     ssize_t line_number{-1};

src/Common/HashTable/FixedClearableHashMap.h

@@ -1,75 +0,0 @@
-#pragma once
-
-#include <Common/HashTable/ClearableHashMap.h>
-#include <Common/HashTable/FixedHashMap.h>
-
-
-template <typename Key, typename TMapped>
-struct FixedClearableHashMapCell
-{
-    using Mapped = TMapped;
-    using State = ClearableHashSetState;
-
-    using value_type = PairNoInit<Key, Mapped>;
-    using mapped_type = Mapped;
-
-    UInt32 version;
-    Mapped mapped;
-
-    FixedClearableHashMapCell() {} /// NOLINT
-    FixedClearableHashMapCell(const Key &, const State & state) : version(state.version) {}
-    FixedClearableHashMapCell(const value_type & value_, const State & state) : version(state.version), mapped(value_.second) {}
-
-    const VoidKey getKey() const { return {}; } /// NOLINT
-    Mapped & getMapped() { return mapped; }
-    const Mapped & getMapped() const { return mapped; }
-
-    bool isZero(const State & state) const { return version != state.version; }
-    void setZero() { version = 0; }
-
-    struct CellExt
-    {
-        CellExt() {} /// NOLINT
-        CellExt(Key && key_, FixedClearableHashMapCell * ptr_) : key(key_), ptr(ptr_) {}
-        void update(Key && key_, FixedClearableHashMapCell * ptr_)
-        {
-            key = key_;
-            ptr = ptr_;
-        }
-        Key key;
-        FixedClearableHashMapCell * ptr;
-        const Key & getKey() const { return key; }
-        Mapped & getMapped() { return ptr->mapped; }
-        const Mapped & getMapped() const { return *ptr->mapped; }
-        const value_type getValue() const { return {key, *ptr->mapped}; } /// NOLINT
-    };
-};
-
-
-template <typename Key, typename Mapped, typename Allocator = HashTableAllocator>
-class FixedClearableHashMap : public FixedHashMap<Key, Mapped, FixedClearableHashMapCell<Key, Mapped>, Allocator>
-{
-public:
-    using Base = FixedHashMap<Key, Mapped, FixedClearableHashMapCell<Key, Mapped>, Allocator>;
-    using Self = FixedClearableHashMap;
-    using LookupResult = typename Base::LookupResult;
-
-    using Base::Base;
-
-    Mapped & operator[](const Key & x)
-    {
-        LookupResult it;
-        bool inserted;
-        this->emplace(x, it, inserted);
-        if (inserted)
-            new (&it->getMapped()) Mapped();
-
-        return it->getMapped();
-    }
-
-    void clear()
-    {
-        ++this->version;
-        this->m_size = 0;
-    }
-};

src/Common/LoggingFormatStringHelpers.h

@@ -106,8 +106,8 @@ template <typename T> constexpr std::string_view tryGetStaticFormatString(T && x
             /// Most likely it was a string literal.
             /// Unfortunately, there's no good way to check if something is a string literal.
             /// But fmtlib requires a format string to be compile-time constant unless fmt::runtime is used.
-            static_assert(std::is_nothrow_convertible<T, const char * const>::value);
-            static_assert(!std::is_pointer<T>::value);
+            static_assert(std::is_nothrow_convertible_v<T, const char * const>);
+            static_assert(!std::is_pointer_v<T>);
             return std::string_view(x);
         }
     }
@@ -127,8 +127,8 @@ template<> struct ConstexprIfsAreNotIfdefs<true>
     {
         /// See tryGetStaticFormatString(...)
         static_assert(!std::is_same_v<std::string, std::decay_t<T>>);
-        static_assert(std::is_nothrow_convertible<T, const char * const>::value);
-        static_assert(!std::is_pointer<T>::value);
+        static_assert(std::is_nothrow_convertible_v<T, const char * const>);
+        static_assert(!std::is_pointer_v<T>);
         return std::string_view(x);
     }
 

src/Common/NamedCollections/NamedCollectionUtils.cpp

@@ -229,6 +229,12 @@ public:
             create_query.changes.emplace_back(name, value);
         create_query.overridability = std::move(result_overridability_map);
 
+        if (create_query.changes.empty())
+            throw Exception(
+                ErrorCodes::BAD_ARGUMENTS,
+                "Named collection cannot be empty (collection name: {})",
+                query.collection_name);
+
         writeCreateQueryToMetadata(
             create_query,
             getMetadataPath(query.collection_name),

src/Common/ProfileEvents.cpp

@@ -317,6 +317,7 @@ The server successfully detected this situation and will download merged part fr
     \
     M(CannotWriteToWriteBufferDiscard, "Number of stack traces dropped by query profiler or signal handler because pipe is full or cannot write to pipe.") \
     M(QueryProfilerSignalOverruns, "Number of times we drop processing of a query profiler signal due to overrun plus the number of signals that OS has not delivered due to overrun.") \
+    M(QueryProfilerConcurrencyOverruns, "Number of times we drop processing of a query profiler signal due to too many concurrent query profilers in other threads, which may indicate overload.") \
     M(QueryProfilerRuns, "Number of times QueryProfiler had been run.") \
     \
     M(CreatedLogEntryForMerge, "Successfully created log entry to merge parts in ReplicatedMergeTree.") \

src/Common/QueryProfiler.cpp

@@ -22,6 +22,7 @@ namespace CurrentMetrics
 namespace ProfileEvents
 {
     extern const Event QueryProfilerSignalOverruns;
+    extern const Event QueryProfilerConcurrencyOverruns;
     extern const Event QueryProfilerRuns;
 }
 
@@ -40,8 +41,19 @@ namespace
     /// to ignore delivered signals after timer_delete().
     thread_local bool signal_handler_disarmed = true;
 
+    /// Don't permit too many threads be busy inside profiler,
+    /// which could slow down the system in some environments.
+    std::atomic<Int64> concurrent_invocations = 0;
+
     void writeTraceInfo(TraceType trace_type, int /* sig */, siginfo_t * info, void * context)
     {
+        SCOPE_EXIT({ concurrent_invocations.fetch_sub(1, std::memory_order_relaxed); });
+        if (concurrent_invocations.fetch_add(1, std::memory_order_relaxed) > 100)
+        {
+            ProfileEvents::incrementNoTrace(ProfileEvents::QueryProfilerConcurrencyOverruns);
+            return;
+        }
+
         auto saved_errno = errno;   /// We must restore previous value of errno in signal handler.
 
 #if defined(OS_LINUX)

src/Common/SensitiveDataMasker.cpp

@@ -1,5 +1,6 @@
 #include "SensitiveDataMasker.h"
 
+#include <mutex>
 #include <set>
 #include <string>
 #include <atomic>
@@ -94,20 +95,28 @@ public:
 SensitiveDataMasker::~SensitiveDataMasker() = default;
 
 std::unique_ptr<SensitiveDataMasker> SensitiveDataMasker::sensitive_data_masker = nullptr;
+std::mutex SensitiveDataMasker::instance_mutex;
 
 void SensitiveDataMasker::setInstance(std::unique_ptr<SensitiveDataMasker> sensitive_data_masker_)
 {
+
     if (!sensitive_data_masker_)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Logical error: the 'sensitive_data_masker' is not set");
 
+    std::lock_guard lock(instance_mutex);
     if (sensitive_data_masker_->rulesCount() > 0)
     {
         sensitive_data_masker = std::move(sensitive_data_masker_);
     }
+    else
+    {
+        sensitive_data_masker.reset();
+    }
 }
 
 SensitiveDataMasker * SensitiveDataMasker::getInstance()
 {
+    std::lock_guard lock(instance_mutex);
     return sensitive_data_masker.get();
 }
 

src/Common/SensitiveDataMasker.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <memory>
+#include <mutex>
 #include <vector>
 #include <cstdint>
 
@@ -45,6 +46,7 @@ class SensitiveDataMasker
 private:
     class MaskingRule;
     std::vector<std::unique_ptr<MaskingRule>> all_masking_rules;
+    static std::mutex instance_mutex;
     static std::unique_ptr<SensitiveDataMasker> sensitive_data_masker;
 
 public:

src/Common/SharedBlockRowRef.h

@@ -1,89 +0,0 @@
-#pragma once
-
-#include <algorithm>
-#include <Core/Block.h>
-#include <Columns/IColumn.h>
-#include <boost/smart_ptr/intrusive_ptr.hpp>
-
-
-namespace DB
-{
-
-/// Allows you refer to the row in the block and hold the block ownership,
-///  and thus avoid creating a temporary row object.
-/// Do not use std::shared_ptr, since there is no need for a place for `weak_count` and `deleter`;
-///  does not use Poco::SharedPtr, since you need to allocate a block and `refcount` in one piece;
-///  does not use Poco::AutoPtr, since it does not have a `move` constructor and there are extra checks for nullptr;
-/// The reference counter is not atomic, since it is used from one thread.
-namespace detail
-{
-    struct SharedBlock : Block
-    {
-        int refcount = 0;
-
-        ColumnRawPtrs all_columns;
-        ColumnRawPtrs sort_columns;
-
-        explicit SharedBlock(Block && block) : Block(std::move(block)) {}
-    };
-}
-
-inline void intrusive_ptr_add_ref(detail::SharedBlock * ptr)
-{
-    ++ptr->refcount;
-}
-
-inline void intrusive_ptr_release(detail::SharedBlock * ptr)
-{
-    if (0 == --ptr->refcount)
-        delete ptr;
-}
-
-using SharedBlockPtr = boost::intrusive_ptr<detail::SharedBlock>;
-
-struct SharedBlockRowRef
-{
-    ColumnRawPtrs * columns = nullptr;
-    size_t row_num = 0;
-    SharedBlockPtr shared_block;
-
-    void swap(SharedBlockRowRef & other)
-    {
-        std::swap(columns, other.columns);
-        std::swap(row_num, other.row_num);
-        std::swap(shared_block, other.shared_block);
-    }
-
-    /// The number and types of columns must match.
-    bool operator==(const SharedBlockRowRef & other) const
-    {
-        size_t size = columns->size();
-        for (size_t i = 0; i < size; ++i)
-            if (0 != (*columns)[i]->compareAt(row_num, other.row_num, *(*other.columns)[i], 1))
-                return false;
-        return true;
-    }
-
-    bool operator!=(const SharedBlockRowRef & other) const
-    {
-        return !(*this == other);
-    }
-
-    void reset()
-    {
-        SharedBlockRowRef empty;
-        swap(empty);
-    }
-
-    bool empty() const { return columns == nullptr; }
-    size_t size() const { return empty() ? 0 : columns->size(); }
-
-    void set(SharedBlockPtr & shared_block_, ColumnRawPtrs * columns_, size_t row_num_)
-    {
-        shared_block = shared_block_;
-        columns = columns_;
-        row_num = row_num_;
-    }
-};
-
-}

src/Common/SimpleActionBlocker.h

@@ -1,79 +0,0 @@
-#pragma once
-#include <atomic>
-
-
-namespace DB
-{
-
-class SimpleActionLock;
-
-
-/// Similar to ActionBlocker, but without weak_ptr magic
-class SimpleActionBlocker
-{
-    using Counter = std::atomic<int>;
-    Counter counter = 0;
-
-public:
-
-    SimpleActionBlocker() = default;
-
-    bool isCancelled() const { return counter > 0; }
-
-    /// Temporarily blocks corresponding actions (while the returned object is alive)
-    friend class SimpleActionLock;
-    inline SimpleActionLock cancel();
-
-    /// Cancel the actions forever.
-    void cancelForever() { ++counter; }
-};
-
-
-/// Blocks related action while a SimpleActionLock instance exists
-class SimpleActionLock
-{
-    SimpleActionBlocker * block = nullptr;
-
-public:
-
-    SimpleActionLock() = default;
-
-    explicit SimpleActionLock(SimpleActionBlocker & block_) : block(&block_)
-    {
-        ++block->counter;
-    }
-
-    SimpleActionLock(const SimpleActionLock &) = delete;
-
-    SimpleActionLock(SimpleActionLock && rhs) noexcept
-    {
-        *this = std::move(rhs);
-    }
-
-    SimpleActionLock & operator=(const SimpleActionLock &) = delete;
-
-    SimpleActionLock & operator=(SimpleActionLock && rhs) noexcept
-    {
-        if (block)
-            --block->counter;
-
-        block = rhs.block;
-        rhs.block = nullptr;
-
-        return *this;
-    }
-
-    ~SimpleActionLock()
-    {
-        if (block)
-            --block->counter;
-    }
-};
-
-
-SimpleActionLock SimpleActionBlocker::cancel()
-{
-    return SimpleActionLock(*this);
-}
-
-}

src/Common/SmallObjectPool.h

@@ -1,52 +0,0 @@
-#pragma once
-
-#include <Common/Arena.h>
-#include <base/unaligned.h>
-
-
-namespace DB
-{
-
-/** Can allocate memory objects of fixed size with deletion support.
-  * For small `object_size`s allocated no less than pointer size.
-  */
-class SmallObjectPool
-{
-private:
-    const size_t object_size;
-    Arena pool;
-    char * free_list = nullptr;
-
-public:
-    explicit SmallObjectPool(size_t object_size_)
-        : object_size{std::max(object_size_, sizeof(char *))}
-    {
-    }
-
-    char * alloc()
-    {
-        if (free_list)
-        {
-            char * res = free_list;
-            free_list = unalignedLoad<char *>(free_list);
-            return res;
-        }
-
-        return pool.alloc(object_size);
-    }
-
-    void free(char * ptr)
-    {
-        unalignedStore<char *>(ptr, free_list);
-        free_list = ptr;
-    }
-
-    /// The size of the allocated pool in bytes
-    size_t size() const
-    {
-        return pool.size();
-    }
-
-};
-
-}

src/Common/config_version.cpp.in

@@ -1,3 +1,17 @@
 /// This file was autogenerated by CMake
 
+#include <Common/config_version.h>
+
+const unsigned VERSION_REVISION = @VERSION_REVISION@;
+const char * VERSION_NAME = "@VERSION_NAME@";
+const unsigned VERSION_MAJOR = @VERSION_MAJOR@;
+const unsigned VERSION_MINOR = @VERSION_MINOR@;
+const unsigned VERSION_PATCH = @VERSION_PATCH@;
+const char * VERSION_STRING = "@VERSION_STRING@";
+const char * VERSION_STRING_SHORT = "@VERSION_STRING_SHORT@";
+const char * VERSION_OFFICIAL = "@VERSION_OFFICIAL@";
+const char * VERSION_FULL = "@VERSION_FULL@";
+const char * VERSION_DESCRIBE = "@VERSION_DESCRIBE@";
+const unsigned VERSION_INTEGER = @VERSION_INTEGER@;
+
 const char * VERSION_GITHASH = "@VERSION_GITHASH@";

src/Common/config_version.h

@@ -0,0 +1,21 @@
+#pragma once
+
+/// These fields are changing only on every release, but we still don't want to have them in the header file,
+/// because it will make build cache less efficient.
+
+// NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
+// only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
+extern const unsigned VERSION_REVISION;
+extern const char * VERSION_NAME;
+extern const unsigned VERSION_MAJOR;
+extern const unsigned VERSION_MINOR;
+extern const unsigned VERSION_PATCH;
+extern const char * VERSION_STRING;
+extern const char * VERSION_STRING_SHORT;
+extern const char * VERSION_OFFICIAL;
+extern const char * VERSION_FULL;
+extern const char * VERSION_DESCRIBE;
+extern const unsigned VERSION_INTEGER;
+
+/// These fields are frequently changing and we don't want to have them in the header file to allow caching.
+extern const char * VERSION_GITHASH;

src/Common/config_version.h.in

@@ -1,24 +0,0 @@
-/// This file was autogenerated by CMake
-
-#pragma once
-
-// NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
-// only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
-#cmakedefine VERSION_REVISION @VERSION_REVISION@
-#cmakedefine VERSION_NAME "@VERSION_NAME@"
-#cmakedefine VERSION_MAJOR @VERSION_MAJOR@
-#cmakedefine VERSION_MINOR @VERSION_MINOR@
-#cmakedefine VERSION_PATCH @VERSION_PATCH@
-#cmakedefine VERSION_STRING "@VERSION_STRING@"
-#cmakedefine VERSION_STRING_SHORT "@VERSION_STRING_SHORT@"
-#cmakedefine VERSION_OFFICIAL "@VERSION_OFFICIAL@"
-#cmakedefine VERSION_FULL "@VERSION_FULL@"
-#cmakedefine VERSION_DESCRIBE "@VERSION_DESCRIBE@"
-#cmakedefine VERSION_INTEGER @VERSION_INTEGER@
-
-/// These fields are frequently changing and we don't want to have them in the header file to allow caching.
-extern const char * VERSION_GITHASH;
-
-#if !defined(VERSION_OFFICIAL)
-#   define VERSION_OFFICIAL ""
-#endif

src/Common/formatIPv6.h

@@ -45,7 +45,7 @@ void formatIPv6(const unsigned char * src, char *& dst, uint8_t zeroed_tail_byte
  * @return            - true if parsed successfully, false otherwise.
  */
 template <typename T, typename EOFfunction>
-requires (std::is_same<typename std::remove_cv<T>::type, char>::value)
+requires (std::is_same_v<std::remove_cv_t<T>, char>)
 inline bool parseIPv4(T * &src, EOFfunction eof, unsigned char * dst, int32_t first_octet = -1)
 {
     if (src == nullptr || first_octet > 255)

src/Common/parseGlobs.cpp

@@ -17,6 +17,11 @@
 
 namespace DB
 {
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+}
+
 /* Transforms string from grep-wildcard-syntax ("{N..M}", "{a,b,c}" as in remote table function and "*", "?") to perl-regexp for using re2 library for matching
  * with such steps:
  * 1) search intervals like {0..9} and enums like {abc,xyz,qwe} in {}, replace them by regexp with pipe (expr1|expr2|expr3),
@@ -116,4 +121,79 @@ std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_glob
     }
     return buf_final_processing.str();
 }
+
+namespace
+{
+void expandSelectorGlobImpl(const std::string & path, std::vector<std::string> & for_match_paths_expanded)
+{
+    /// regexp for {expr1,expr2,....} (a selector glob);
+    /// expr1, expr2,... cannot contain any of these: '{', '}', ','
+    static const re2::RE2 selector_regex(R"({([^{}*,]+,[^{}*]*[^{}*,])})");
+
+    std::string_view path_view(path);
+    std::string_view matched;
+
+    // No (more) selector globs found, quit
+    if (!RE2::FindAndConsume(&path_view, selector_regex, &matched))
+    {
+        for_match_paths_expanded.push_back(path);
+        return;
+    }
+
+    std::vector<size_t> anchor_positions;
+    bool opened = false;
+    bool closed = false;
+
+    // Looking for first occurrence of {} selector: write down positions of {, } and all intermediate commas
+    for (auto it = path.begin(); it != path.end(); ++it)
+    {
+        if (*it == '{')
+        {
+            if (opened)
+                throw Exception(ErrorCodes::BAD_ARGUMENTS,
+                                "Unexpected '{{' found in path '{}' at position {}.", path, it - path.begin());
+            anchor_positions.push_back(std::distance(path.begin(), it));
+            opened = true;
+        }
+        else if (*it == '}')
+        {
+            if (!opened)
+                throw Exception(ErrorCodes::BAD_ARGUMENTS,
+                                "Unexpected '}}' found in path '{}' at position {}.", path, it - path.begin());
+            anchor_positions.push_back(std::distance(path.begin(), it));
+            closed = true;
+            break;
+        }
+        else if (*it == ',')
+        {
+            if (!opened)
+                throw Exception(ErrorCodes::BAD_ARGUMENTS,
+                                "Unexpected ',' found in path '{}' at position {}.", path, std::distance(path.begin(), it));
+            anchor_positions.push_back(std::distance(path.begin(), it));
+        }
+    }
+    if (!opened || !closed)
+        throw Exception(ErrorCodes::BAD_ARGUMENTS,
+                        "Invalid {{}} glob in path {}.", path);
+
+    // generate result: prefix/{a,b,c}/suffix -> [prefix/a/suffix, prefix/b/suffix, prefix/c/suffix]
+    std::string common_prefix = path.substr(0, anchor_positions.front());
+    std::string common_suffix = path.substr(anchor_positions.back() + 1);
+    for (size_t i = 1; i < anchor_positions.size(); ++i)
+    {
+        std::string current_selection =
+                path.substr(anchor_positions[i-1] + 1, (anchor_positions[i] - anchor_positions[i-1] - 1));
+
+        std::string expanded_matcher = common_prefix + current_selection + common_suffix;
+        expandSelectorGlobImpl(expanded_matcher, for_match_paths_expanded);
+    }
+}
+}
+
+std::vector<std::string> expandSelectionGlob(const std::string & path)
+{
+    std::vector<std::string> result;
+    expandSelectorGlobImpl(path, result);
+    return result;
+}
 }

src/Common/parseGlobs.h

@@ -4,7 +4,11 @@
 
 namespace DB
 {
-/* Parse globs in string and make a regexp for it.
- */
-std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_globs);
+    /// Parse globs in string and make a regexp for it.
+    std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_globs);
+
+    /// Process {a,b,c...} globs:
+    /// Don't match it against regex, but generate a,b,c strings instead and process each of them separately.
+    /// E.g. for a string like `file{1,2,3}.csv` return vector of strings: {`file1.csv`,`file2.csv`,`file3.csv`}
+    std::vector<std::string> expandSelectionGlob(const std::string & path);
 }

src/Compression/CompressionCodecDeflateQpl.cpp

@@ -377,7 +377,7 @@ uint8_t CompressionCodecDeflateQpl::getMethodByte() const
 
 void CompressionCodecDeflateQpl::updateHash(SipHash & hash) const
 {
-    getCodecDesc()->updateTreeHash(hash);
+    getCodecDesc()->updateTreeHash(hash, /*ignore_aliases=*/ true);
 }
 
 UInt32 CompressionCodecDeflateQpl::getMaxCompressedDataSize(UInt32 uncompressed_size) const

src/Compression/CompressionCodecDelta.cpp

@@ -56,7 +56,7 @@ uint8_t CompressionCodecDelta::getMethodByte() const
 
 void CompressionCodecDelta::updateHash(SipHash & hash) const
 {
-    getCodecDesc()->updateTreeHash(hash);
+    getCodecDesc()->updateTreeHash(hash, /*ignore_aliases=*/ true);
 }
 
 namespace
@@ -66,7 +66,7 @@ template <typename T>
 void compressDataForType(const char * source, UInt32 source_size, char * dest)
 {
     if (source_size % sizeof(T) != 0)
-        throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot delta compress, data size {} is not aligned to {}", source_size, sizeof(T));
+        throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress with Delta codec, data size {} is not aligned to {}", source_size, sizeof(T));
 
     T prev_src = 0;
     const char * const source_end = source + source_size;
@@ -87,7 +87,7 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest,
     const char * const output_end = dest + output_size;
 
     if (source_size % sizeof(T) != 0)
-        throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot delta decompress, data size {} is not aligned to {}", source_size, sizeof(T));
+        throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress Delta-encoded data, data size {} is not aligned to {}", source_size, sizeof(T));
 
     T accumulator{};
     const char * const source_end = source + source_size;
@@ -95,7 +95,7 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest,
     {
         accumulator += unalignedLoadLittleEndian<T>(source);
         if (dest + sizeof(accumulator) > output_end) [[unlikely]]
-            throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress the data");
+            throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress delta-encoded data");
         unalignedStoreLittleEndian<T>(dest, accumulator);
 
         source += sizeof(T);
@@ -133,7 +133,7 @@ UInt32 CompressionCodecDelta::doCompressData(const char * source, UInt32 source_
 void CompressionCodecDelta::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const
 {
     if (source_size < 2)
-        throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress. File has wrong header");
+        throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress delta-encoded data. File has wrong header");
 
     if (uncompressed_size == 0)
         return;
@@ -141,13 +141,13 @@ void CompressionCodecDelta::doDecompressData(const char * source, UInt32 source_
     UInt8 bytes_size = source[0];
 
     if (!(bytes_size == 1 || bytes_size == 2 || bytes_size == 4 || bytes_size == 8))
-        throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress. File has wrong header");
+        throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress delta-encoded data. File has wrong header");
 
     UInt8 bytes_to_skip = uncompressed_size % bytes_size;
     UInt32 output_size = uncompressed_size - bytes_to_skip;
 
     if (static_cast<UInt32>(2 + bytes_to_skip) > source_size)
-        throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress. File has wrong header");
+        throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress delta-encoded data. File has wrong header");
 
     memcpy(dest, &source[2], bytes_to_skip);
     UInt32 source_size_no_header = source_size - bytes_to_skip - 2;