Merge remote-tracking branch 'origin/master' into query-result-cache

.github/workflows/backport_branches.yml

@@ -12,11 +12,10 @@ jobs:
   PythonUnitTests:
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Python unit tests
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
@@ -24,34 +23,32 @@ jobs:
   DockerHubPushAarch64:
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Images check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_images_check.py --suffix aarch64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images_aarch64
           path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
   DockerHubPushAmd64:
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Images check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_images_check.py --suffix amd64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images_amd64
           path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
@@ -59,18 +56,17 @@ jobs:
     needs: [DockerHubPushAmd64, DockerHubPushAarch64]
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Download changed aarch64 images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images_aarch64
           path: ${{ runner.temp }}
       - name: Download changed amd64 images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images_amd64
           path: ${{ runner.temp }}
@@ -79,7 +75,7 @@ jobs:
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images
           path: ${{ runner.temp }}/changed_images.json
@@ -94,13 +90,12 @@ jobs:
           REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
           REPORTS_PATH=${{runner.temp}}/reports_dir
           EOF
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Download json reports
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
       - name: CompatibilityCheck
@@ -132,28 +127,25 @@ jobs:
           BUILD_NAME=package_release
           EOF
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.IMAGES_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
+          submodules: true
           fetch-depth: 0 # For a proper version and performance artifacts
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
-          git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
           cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
       - name: Upload build URLs to artifacts
         if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.BUILD_URLS }}
           path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
@@ -177,28 +169,25 @@ jobs:
           BUILD_NAME=package_aarch64
           EOF
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.IMAGES_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
+          submodules: true
           fetch-depth: 0 # For a proper version and performance artifacts
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
-          git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
           cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
       - name: Upload build URLs to artifacts
         if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.BUILD_URLS }}
           path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
@@ -222,26 +211,24 @@ jobs:
           BUILD_NAME=package_asan
           EOF
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.IMAGES_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          submodules: true
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
-          git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
           cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
       - name: Upload build URLs to artifacts
         if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.BUILD_URLS }}
           path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
@@ -265,26 +252,24 @@ jobs:
           BUILD_NAME=package_tsan
           EOF
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.IMAGES_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          submodules: true
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
-          git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
           cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
       - name: Upload build URLs to artifacts
         if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.BUILD_URLS }}
           path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
@@ -308,26 +293,24 @@ jobs:
           BUILD_NAME=package_debug
           EOF
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.IMAGES_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          submodules: true
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
-          git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
           cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
       - name: Upload build URLs to artifacts
         if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.BUILD_URLS }}
           path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
@@ -351,28 +334,25 @@ jobs:
           BUILD_NAME=binary_darwin
           EOF
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.IMAGES_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
+          submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
-          git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
           cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
       - name: Upload build URLs to artifacts
         if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.BUILD_URLS }}
           path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
@@ -396,28 +376,25 @@ jobs:
           BUILD_NAME=binary_darwin_aarch64
           EOF
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.IMAGES_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
+          submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
-          git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
           cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
       - name: Upload build URLs to artifacts
         if: ${{ success() || failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.BUILD_URLS }}
           path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
@@ -436,12 +413,10 @@ jobs:
       - BuilderDebAarch64
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
           fetch-depth: 0  # It MUST BE THE SAME for all dependencies and the job itself
       - name: Check docker clickhouse/clickhouse-server building
         run: |
@@ -477,14 +452,13 @@ jobs:
           NEEDS_DATA_PATH=${{runner.temp}}/needs.json
           EOF
       - name: Download json reports
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Report Builder
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -516,14 +490,13 @@ jobs:
           NEEDS_DATA_PATH=${{runner.temp}}/needs.json
           EOF
       - name: Download json reports
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Report Builder
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -556,14 +529,13 @@ jobs:
           KILL_TIMEOUT=10800
           EOF
       - name: Download json reports
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Functional test
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -594,14 +566,13 @@ jobs:
           KILL_TIMEOUT=3600
           EOF
       - name: Download json reports
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Functional test
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -635,14 +606,13 @@ jobs:
           REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse
           EOF
       - name: Download json reports
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Stress test
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -672,14 +642,13 @@ jobs:
           REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
           EOF
       - name: Download json reports
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Integration test
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -706,11 +675,10 @@ jobs:
       - CompatibilityCheck
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Finish label
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"

.github/workflows/cherry_pick.yml

@@ -28,8 +28,9 @@ jobs:
           REPO_TEAM=core
           EOF
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
           fetch-depth: 0
       - name: Cherry pick

.github/workflows/docs_check.yml

@@ -16,15 +16,15 @@ on:  # yamllint disable-line rule:truthy
       - 'docker/docs/**'
       - 'docs/**'
       - 'website/**'
+      - 'utils/check-style/aspell-ignore/**'
 jobs:
   CheckLabels:
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -rf "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Labels check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
@@ -33,17 +33,16 @@ jobs:
     needs: CheckLabels
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Images check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_images_check.py --suffix aarch64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images_aarch64
           path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
@@ -51,17 +50,16 @@ jobs:
     needs: CheckLabels
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Images check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_images_check.py --suffix amd64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images_amd64
           path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
@@ -69,18 +67,17 @@ jobs:
     needs: [DockerHubPushAmd64, DockerHubPushAarch64]
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Download changed aarch64 images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images_aarch64
           path: ${{ runner.temp }}
       - name: Download changed amd64 images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images_amd64
           path: ${{ runner.temp }}
@@ -89,7 +86,7 @@ jobs:
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images
           path: ${{ runner.temp }}/changed_images.json
@@ -109,15 +106,14 @@ jobs:
       - name: Download changed images
         # even if artifact does not exist, e.g. on `do not test` label or failed Docker job
         continue-on-error: true
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.TEMP_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Style Check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
@@ -139,15 +135,14 @@ jobs:
           REPO_COPY=${{runner.temp}}/docs_check/ClickHouse
           EOF
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.TEMP_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -rf "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Docs Check
         run: |
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -166,11 +161,10 @@ jobs:
       - DocsCheck
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Finish label
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"

.github/workflows/docs_release.yml

@@ -17,39 +17,38 @@ concurrency:
       - 'docs/**'
       - 'utils/list-versions/version_date.tsv'
       - 'website/**'
+      - 'utils/check-style/aspell-ignore/**'
   workflow_dispatch:
 jobs:
   DockerHubPushAarch64:
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Images check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_images_check.py --suffix aarch64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images_aarch64
           path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
   DockerHubPushAmd64:
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Images check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_images_check.py --suffix amd64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images_amd64
           path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
@@ -57,18 +56,17 @@ jobs:
     needs: [DockerHubPushAmd64, DockerHubPushAarch64]
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Download changed aarch64 images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images_aarch64
           path: ${{ runner.temp }}
       - name: Download changed amd64 images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images_amd64
           path: ${{ runner.temp }}
@@ -77,7 +75,7 @@ jobs:
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images
           path: ${{ runner.temp }}/changed_images.json
@@ -96,13 +94,12 @@ jobs:
           ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
           RCSK
           EOF
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.TEMP_PATH }}

.github/workflows/jepsen.yml

@@ -19,12 +19,10 @@ jobs:
           TEMP_PATH=${{runner.temp}}/keeper_jepsen
           REPO_COPY=${{runner.temp}}/keeper_jepsen/ClickHouse
           EOF
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
           fetch-depth: 0
       - name: Jepsen Test
         run: |
@@ -50,12 +48,10 @@ jobs:
   #         TEMP_PATH=${{runner.temp}}/server_jepsen
   #         REPO_COPY=${{runner.temp}}/server_jepsen/ClickHouse
   #         EOF
-  #     - name: Clear repository
-  #       run: |
-  #         sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
   #     - name: Check out repository code
-  #       uses: actions/checkout@v2
+  #       uses: ClickHouse/checkout@v1
   #       with:
+  #         clear-repository: true
   #         fetch-depth: 0
   #     - name: Jepsen Test
   #       run: |

.github/workflows/nightly.yml

@@ -16,34 +16,32 @@ jobs:
   DockerHubPushAarch64:
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Images check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_images_check.py --suffix aarch64 --all
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images_aarch64
           path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
   DockerHubPushAmd64:
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Images check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_images_check.py --suffix amd64 --all
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images_amd64
           path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
@@ -51,18 +49,17 @@ jobs:
     needs: [DockerHubPushAmd64, DockerHubPushAarch64]
     runs-on: [self-hosted, style-checker]
     steps:
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
       - name: Download changed aarch64 images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images_aarch64
           path: ${{ runner.temp }}
       - name: Download changed amd64 images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images_amd64
           path: ${{ runner.temp }}
@@ -71,7 +68,7 @@ jobs:
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
       - name: Upload images files to artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: changed_images
           path: ${{ runner.temp }}/changed_images.json
@@ -90,22 +87,17 @@ jobs:
           EOF
           echo "COVERITY_TOKEN=${{ secrets.COVERITY_TOKEN }}" >> "$GITHUB_ENV"
       - name: Download changed images
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: changed_images
           path: ${{ env.IMAGES_PATH }}
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        id: coverity-checkout
+        uses: ClickHouse/checkout@v1
-        uses: actions/checkout@v2
         with:
-          fetch-depth: 0 # otherwise we will have no info about contributors
+          clear-repository: true
+          submodules: true
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
-          git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -134,8 +126,10 @@ jobs:
       CC: clang-15
       CXX: clang++-15
     steps:
-      - uses: actions/checkout@v2
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
           submodules: true
       - name: Set up JDK 11

.github/workflows/release.yml

@@ -20,7 +20,7 @@ jobs:
         REPO_COPY=${{runner.temp}}/release_packages/ClickHouse
         EOF
     - name: Check out repository code
-      uses: actions/checkout@v2
+      uses: ClickHouse/checkout@v1
       with:
         # Always use the most recent script version
         ref: master
@@ -50,12 +50,10 @@ jobs:
   DockerServerImages:
     runs-on: [self-hosted, style-checker]
     steps:
-    - name: Clear repository
-      run: |
-        sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
     - name: Check out repository code
-      uses: actions/checkout@v2
+      uses: ClickHouse/checkout@v1
       with:
+        clear-repository: true
         fetch-depth: 0  # otherwise we will have no version info
     - name: Check docker clickhouse/clickhouse-server building
       run: |

.github/workflows/tags_stable.yml

@@ -34,7 +34,7 @@ jobs:
       run: |
         echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV"
     - name: Check out repository code
-      uses: actions/checkout@v2
+      uses: ClickHouse/checkout@v1
       with:
         ref: master
         fetch-depth: 0

.github/workflows/woboq.yml

@@ -21,12 +21,10 @@ jobs:
           REPO_COPY=${{runner.temp}}/codebrowser/ClickHouse
           IMAGES_PATH=${{runner.temp}}/images_path
           EOF
-      - name: Clear repository
-        run: |
-          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: ClickHouse/checkout@v1
         with:
+          clear-repository: true
           submodules: 'true'
       - name: Codebrowser
         run: |

.gitmodules

@@ -269,9 +269,6 @@
 [submodule "contrib/vectorscan"]
 	path = contrib/vectorscan
 	url = https://github.com/VectorCamp/vectorscan.git
-[submodule "contrib/liburing"]
-	path = contrib/liburing
-	url = https://github.com/axboe/liburing.git
 [submodule "contrib/c-ares"]
 	path = contrib/c-ares
 	url = https://github.com/ClickHouse/c-ares

CHANGELOG.md

@@ -17,6 +17,9 @@
 
 ### <a id="2212"></a> ClickHouse release 22.12, 2022-12-15
 
+#### Backward Incompatible Change
+* Add `GROUP BY ALL` syntax: [#37631](https://github.com/ClickHouse/ClickHouse/issues/37631). [#42265](https://github.com/ClickHouse/ClickHouse/pull/42265) ([刘陶峰](https://github.com/taofengliu)). If you have a column or an alias named `all` and doing `GROUP BY all` without the intention to group by all the columns, the query will have a different semantic. To keep the old semantic, put `all` into backticks or double quotes `"all"` to make it an identifier instead of a keyword.
+
 #### Upgrade Notes
 * Fixed backward incompatibility in (de)serialization of states of `min`, `max`, `any*`, `argMin`, `argMax` aggregate functions with `String` argument. The incompatibility affects 22.9, 22.10 and 22.11 branches (fixed since 22.9.6, 22.10.4 and 22.11.2 correspondingly). Some minor releases of 22.3, 22.7 and 22.8 branches are also affected: 22.3.13...22.3.14 (fixed since 22.3.15), 22.8.6...22.8.9 (fixed since 22.8.10), 22.7.6 and newer (will not be fixed in 22.7, we recommend upgrading from 22.7.* to 22.8.10 or newer). This release note does not concern users that have never used affected versions. Incompatible versions append an extra `'\0'` to strings when reading states of the aggregate functions mentioned above. For example, if an older version saved state of `anyState('foobar')` to `state_column` then the incompatible version will print `'foobar\0'` on `anyMerge(state_column)`. Also incompatible versions write states of the aggregate functions without trailing `'\0'`. Newer versions (that have the fix) can correctly read data written by all versions including incompatible versions, except one corner case. If an incompatible version saved a state with a string that actually ends with null character, then newer version will trim trailing `'\0'` when reading state of affected aggregate function. For example, if an incompatible version saved state of `anyState('abrac\0dabra\0')` to `state_column` then newer versions will print `'abrac\0dabra'` on `anyMerge(state_column)`. The issue also affects distributed queries when an incompatible version works in a cluster together with older or newer versions. [#43038](https://github.com/ClickHouse/ClickHouse/pull/43038) ([Alexander Tokmakov](https://github.com/tavplubix), [Raúl Marín](https://github.com/Algunenano)). Note: all the official ClickHouse builds already include the patches. This is not necessarily true for unofficial third-party builds that should be avoided.
 

CMakeLists.txt

@@ -377,15 +377,15 @@ set (DEBUG_INFO_FLAGS "-g -gdwarf-4")
 
 set (CMAKE_CXX_FLAGS                     "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS}")
 set (CMAKE_CXX_FLAGS_RELWITHDEBINFO      "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
-set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
+set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
 
 set (CMAKE_C_FLAGS                       "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${CMAKE_C_FLAGS_ADD}")
 set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}")
-set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_C_FLAGS_ADD}")
+set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}")
 
 set (CMAKE_ASM_FLAGS                     "${CMAKE_ASM_FLAGS} ${COMPILER_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
 set (CMAKE_ASM_FLAGS_RELWITHDEBINFO      "${CMAKE_ASM_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
-set (CMAKE_ASM_FLAGS_DEBUG               "${CMAKE_ASM_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_ASM_FLAGS_ADD}")
+set (CMAKE_ASM_FLAGS_DEBUG               "${CMAKE_ASM_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
 
 if (COMPILER_CLANG)
     if (OS_DARWIN)

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2016-2022 ClickHouse, Inc.
+Copyright 2016-2023 ClickHouse, Inc.
 
                                  Apache License
                            Version 2.0, January 2004
@@ -188,7 +188,7 @@ Copyright 2016-2022 ClickHouse, Inc.
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2016-2022 ClickHouse, Inc.
+   Copyright 2016-2023 ClickHouse, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -16,6 +16,6 @@ ClickHouse® is an open-source column-oriented database management system that a
 * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
 
 ## Upcoming events
-* [**v22.12 Release Webinar**](https://clickhouse.com/company/events/v22-12-release-webinar) 22.12 is the ClickHouse Christmas release. There are plenty of gifts (a new JOIN algorithm among them) and we adopted something from MongoDB. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release. 
+* **Recording available**: [**v22.12 Release Webinar**](https://www.youtube.com/watch?v=sREupr6uc2k) 22.12 is the ClickHouse Christmas release. There are plenty of gifts (a new JOIN algorithm among them) and we adopted something from MongoDB. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
 * [**ClickHouse Meetup at the CHEQ office in Tel Aviv**](https://www.meetup.com/clickhouse-tel-aviv-user-group/events/289599423/) - Jan 16 - We are very excited to be holding our next in-person ClickHouse meetup at the CHEQ office in Tel Aviv! Hear from CHEQ, ServiceNow and Contentsquare, as well as a deep dive presentation from ClickHouse CTO Alexey Milovidov. Join us for a fun evening of talks, food and discussion!
 * [**ClickHouse Meetup at Microsoft Office in Seattle**](https://www.meetup.com/clickhouse-seattle-user-group/events/290310025/) - Jan 18 - Keep an eye on this space as we will be announcing speakers soon!

base/base/BorrowedObjectPool.h

@@ -10,7 +10,7 @@
 #include <base/MoveOrCopyIfThrow.h>
 
 /** Pool for limited size objects that cannot be used from different threads simultaneously.
-  * The main use case is to have fixed size of objects that can be reused in difference threads during their lifetime
+  * The main use case is to have fixed size of objects that can be reused in different threads during their lifetime
   * and have to be initialized on demand.
   * Two main properties of pool are allocated objects size and borrowed objects size.
   * Allocated objects size is size of objects that are currently allocated by the pool.

base/base/CMakeLists.txt

@@ -8,16 +8,13 @@ set (SRCS
     getPageSize.cpp
     getThreadId.cpp
     JSON.cpp
-    LineReader.cpp
     mremap.cpp
     phdr_cache.cpp
     preciseExp10.cpp
-    setTerminalEcho.cpp
     shift10.cpp
     sleep.cpp
     terminalColors.cpp
     errnoToString.cpp
-    ReplxxLineReader.cpp
     StringRef.cpp
     safeExit.cpp
     throwError.cpp
@@ -40,11 +37,6 @@ else ()
     target_compile_definitions(common PUBLIC WITH_COVERAGE=0)
 endif ()
 
-# FIXME: move libraries for line reading out from base
-if (TARGET ch_rust::skim)
-    target_link_libraries(common PUBLIC ch_rust::skim)
-endif()
-
 target_include_directories(common PUBLIC .. "${CMAKE_CURRENT_BINARY_DIR}/..")
 
 if (OS_DARWIN AND NOT USE_STATIC_LIBRARIES)

base/base/setTerminalEcho.cpp

@@ -1,28 +0,0 @@
-#include <base/setTerminalEcho.h>
-#include <base/errnoToString.h>
-#include <stdexcept>
-#include <cstring>
-#include <string>
-#include <termios.h>
-#include <unistd.h>
-
-
-void setTerminalEcho(bool enable)
-{
-    /// Obtain terminal attributes,
-    /// toggle the ECHO flag
-    /// and set them back.
-
-    struct termios tty{};
-
-    if (0 != tcgetattr(STDIN_FILENO, &tty))
-        throw std::runtime_error(std::string("setTerminalEcho failed get: ") + errnoToString());
-
-    if (enable)
-        tty.c_lflag |= ECHO;
-    else
-        tty.c_lflag &= ~ECHO;
-
-    if (0 != tcsetattr(STDIN_FILENO, TCSANOW, &tty))
-        throw std::runtime_error(std::string("setTerminalEcho failed set: ") + errnoToString());
-}

base/base/setTerminalEcho.h

@@ -1,4 +0,0 @@
-#pragma once
-
-/// Enable or disable echoing of typed characters. Throws std::runtime_error on error.
-void setTerminalEcho(bool enable);

contrib/cctz

@@ -1 +1 @@
-Subproject commit 5c8528fb35e89ee0b3a7157490423fba0d4dd7b5
+Subproject commit 7c78edd52b4d65acc103c2f195818ffcabe6fe0d

contrib/libunwind-cmake/CMakeLists.txt

@@ -43,7 +43,10 @@ set_target_properties(unwind PROPERTIES FOLDER "contrib/libunwind-cmake")
 
 target_include_directories(unwind SYSTEM BEFORE PUBLIC $<BUILD_INTERFACE:${LIBUNWIND_SOURCE_DIR}/include>)
 target_compile_definitions(unwind PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY)
-target_compile_options(unwind PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all $<$<COMPILE_LANGUAGE:CXX>:-nostdinc++ -fno-rtti>)
+
+# We should enable optimizations (otherwise it will be too slow in debug)
+# and disable sanitizers (otherwise infinite loop may happen)
+target_compile_options(unwind PRIVATE -O3 -fno-exceptions -funwind-tables -fno-sanitize=all $<$<COMPILE_LANGUAGE:CXX>:-nostdinc++ -fno-rtti>)
 
 check_c_compiler_flag(-Wunused-but-set-variable HAVE_WARNING_UNUSED_BUT_SET_VARIABLE)
 if (HAVE_WARNING_UNUSED_BUT_SET_VARIABLE)

contrib/sysroot

@@ -1 +1 @@
-Subproject commit e9fb375d0a1e5ebfd74c043f088f2342552103f8
+Subproject commit 0f41651860fa4a530ecd68b93a15b8fd77397adf

docker/packager/packager

@@ -131,7 +131,7 @@ def parse_env_variables(
     ARM_V80COMPAT_SUFFIX = "-aarch64-v80compat"
     FREEBSD_SUFFIX = "-freebsd"
     PPC_SUFFIX = "-ppc64le"
-    AMD64_SSE2_SUFFIX = "-amd64sse2"
+    AMD64_COMPAT_SUFFIX = "-amd64-compat"
 
     result = []
     result.append("OUTPUT_DIR=/output")
@@ -144,7 +144,7 @@ def parse_env_variables(
     is_cross_arm_v80compat = compiler.endswith(ARM_V80COMPAT_SUFFIX)
     is_cross_ppc = compiler.endswith(PPC_SUFFIX)
     is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
-    is_amd64_sse2 = compiler.endswith(AMD64_SSE2_SUFFIX)
+    is_amd64_compat = compiler.endswith(AMD64_COMPAT_SUFFIX)
 
     if is_cross_darwin:
         cc = compiler[: -len(DARWIN_SUFFIX)]
@@ -197,8 +197,8 @@ def parse_env_variables(
         cmake_flags.append(
             "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-ppc64le.cmake"
         )
-    elif is_amd64_sse2:
+    elif is_amd64_compat:
-        cc = compiler[: -len(AMD64_SSE2_SUFFIX)]
+        cc = compiler[: -len(AMD64_COMPAT_SUFFIX)]
         result.append("DEB_ARCH=amd64")
         cmake_flags.append("-DNO_SSE3_OR_HIGHER=1")
     else:
@@ -358,7 +358,7 @@ if __name__ == "__main__":
             "clang-15-aarch64",
             "clang-15-aarch64-v80compat",
             "clang-15-ppc64le",
-            "clang-15-amd64sse2",
+            "clang-15-amd64-compat",
             "clang-15-freebsd",
             "gcc-11",
         ),

docker/server/entrypoint.sh

@@ -80,7 +80,7 @@ do
 done
 
 # if clickhouse user is defined - create it (user "default" already exists out of box)
-if [ -n "$CLICKHOUSE_USER" ] && [ "$CLICKHOUSE_USER" != "default" ] || [ -n "$CLICKHOUSE_PASSWORD" ]; then
+if [ -n "$CLICKHOUSE_USER" ] && [ "$CLICKHOUSE_USER" != "default" ] || [ -n "$CLICKHOUSE_PASSWORD" ] || [ "$CLICKHOUSE_ACCESS_MANAGEMENT" != "0" ]; then
     echo "$0: create new user '$CLICKHOUSE_USER' instead 'default'"
     cat <<EOT > /etc/clickhouse-server/users.d/default-user.xml
     <clickhouse>
@@ -120,8 +120,8 @@ if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then
     pid="$!"
 
     # check if clickhouse is ready to accept connections
-    # will try to send ping clickhouse via http_port (max 12 retries by default, with 1 sec timeout and 1 sec delay between retries)
+    # will try to send ping clickhouse via http_port (max 1000 retries by default, with 1 sec timeout and 1 sec delay between retries)
-    tries=${CLICKHOUSE_INIT_TIMEOUT:-12}
+    tries=${CLICKHOUSE_INIT_TIMEOUT:-1000}
     while ! wget --spider --no-check-certificate -T 1 -q "$URL" 2>/dev/null; do
         if [ "$tries" -le "0" ]; then
             echo >&2 'ClickHouse init process failed.'

docker/test/fuzzer/query-fuzzer-tweaks-users.xml

@@ -2,6 +2,7 @@
     <profiles>
         <default>
             <max_execution_time>10</max_execution_time>
+
             <!--
                 Don't let the fuzzer change this setting (I've actually seen it
                 do this before).
@@ -14,6 +15,11 @@
                 <max_memory_usage>
                     <max>10G</max>
                 </max_memory_usage>
+
+                <!-- Analyzer is unstable, not ready for testing. -->
+                <allow_experimental_analyzer>
+                    <readonly/>
+                </allow_experimental_analyzer>
             </constraints>
         </default>
     </profiles>

docker/test/fuzzer/run-fuzzer.sh

@@ -51,7 +51,6 @@ function clone
     )
 
     ls -lath ||:
-
 }
 
 function wget_with_retry
@@ -75,6 +74,7 @@ function download
     ./clickhouse ||:
     ln -s ./clickhouse ./clickhouse-server
     ln -s ./clickhouse ./clickhouse-client
+    ln -s ./clickhouse ./clickhouse-local
 
     # clickhouse-server is in the current dir
     export PATH="$PWD:$PATH"
@@ -91,6 +91,12 @@ function configure
     cp -av --dereference "$script_dir"/query-fuzzer-tweaks-users.xml db/users.d
     cp -av --dereference "$script_dir"/allow-nullable-key.xml db/config.d
 
+    cat > db/config.d/max_server_memory_usage_to_ram_ratio.xml <<EOL
+<clickhouse>
+    <max_server_memory_usage_to_ram_ratio>0.75</max_server_memory_usage_to_ram_ratio>
+</clickhouse>
+EOL
+
     cat > db/config.d/core.xml <<EOL
 <clickhouse>
     <core_dump>
@@ -151,7 +157,7 @@ function fuzz
     mkdir -p /var/run/clickhouse-server
 
     # NOTE: we use process substitution here to preserve keep $! as a pid of clickhouse-server
-    clickhouse-server --config-file db/config.xml --pid-file /var/run/clickhouse-server/clickhouse-server.pid -- --path db  2>&1 | pigz > server.log.gz &
+    clickhouse-server --config-file db/config.xml --pid-file /var/run/clickhouse-server/clickhouse-server.pid -- --path db > server.log 2>&1 &
     server_pid=$!
 
     kill -0 $server_pid
@@ -256,12 +262,21 @@ quit
     if [ "$server_died" == 1 ]
     then
         # The server has died.
-        task_exit_code=210
+        if ! grep --text -ao "Received signal.*\|Logical error.*\|Assertion.*failed\|Failed assertion.*\|.*runtime error: .*\|.*is located.*\|SUMMARY: AddressSanitizer:.*\|SUMMARY: MemorySanitizer:.*\|SUMMARY: ThreadSanitizer:.*\|.*_LIBCPP_ASSERT.*" server.log > description.txt
-        echo "failure" > status.txt
-        if ! zgrep --text -ao "Received signal.*\|Logical error.*\|Assertion.*failed\|Failed assertion.*\|.*runtime error: .*\|.*is located.*\|SUMMARY: AddressSanitizer:.*\|SUMMARY: MemorySanitizer:.*\|SUMMARY: ThreadSanitizer:.*\|.*_LIBCPP_ASSERT.*" server.log.gz > description.txt
         then
             echo "Lost connection to server. See the logs." > description.txt
         fi
+
+        if grep -F --text 'Sanitizer: out-of-memory' description.txt
+        then
+            # OOM of sanitizer is not a problem we can handle - treat it as success, but preserve the description.
+            task_exit_code=0
+            echo "success" > status.txt
+        else
+            task_exit_code=210
+            echo "failure" > status.txt
+        fi
+
     elif [ "$fuzzer_exit_code" == "143" ] || [ "$fuzzer_exit_code" == "0" ]
     then
         # Variants of a normal run:
@@ -327,24 +342,28 @@ case "$stage" in
     time fuzz
     ;&
 "report")
+
 CORE_LINK=''
 if [ -f core.gz ]; then
     CORE_LINK='<a href="core.gz">core.gz</a>'
 fi
+
+grep --text -F '<Fatal>' server.log > fatal.log ||:
+
+pigz server.log
+
 cat > report.html <<EOF ||:
 <!DOCTYPE html>
 <html lang="en">
   <style>
 body { font-family: "DejaVu Sans", "Noto Sans", Arial, sans-serif; background: #EEE; }
 h1 { margin-left: 10px; }
-th, td { border: 0; padding: 5px 10px 5px 10px; text-align: left; vertical-align: top; line-height: 1.5; background-color: #FFF;
+th, td { border: 0; padding: 5px 10px 5px 10px; text-align: left; vertical-align: top; line-height: 1.5; background-color: #FFF; }
-td { white-space: pre; font-family: Monospace, Courier New; }
+td { white-space: pre; font-family: Monospace, Courier New; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }
-border: 0; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }
 a { color: #06F; text-decoration: none; }
 a:hover, a:active { color: #F40; text-decoration: underline; }
 table { border: 0; }
 p.links a { padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-space: nowrap; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }
-th { cursor: pointer; }
 
   </style>
   <title>AST Fuzzer for PR #${PR_TO_TEST} @ ${SHA_TO_TEST}</title>
@@ -352,17 +371,32 @@ th { cursor: pointer; }
 <body>
 <div class="main">
 
-<h1>AST Fuzzer for PR #${PR_TO_TEST} @ ${SHA_TO_TEST}</h1>
+<h1>AST Fuzzer for PR <a href="https://github.com/ClickHouse/ClickHouse/pull/${PR_TO_TEST}">#${PR_TO_TEST}</a> @ ${SHA_TO_TEST}</h1>
 <p class="links">
-<a href="runlog.log">runlog.log</a>
+  <a href="run.log">run.log</a>
-<a href="fuzzer.log">fuzzer.log</a>
+  <a href="fuzzer.log">fuzzer.log</a>
-<a href="server.log.gz">server.log.gz</a>
+  <a href="server.log.gz">server.log.gz</a>
-<a href="main.log">main.log</a>
+  <a href="main.log">main.log</a>
-${CORE_LINK}
+  ${CORE_LINK}
 </p>
 <table>
-<tr><th>Test name</th><th>Test status</th><th>Description</th></tr>
+<tr>
-<tr><td>AST Fuzzer</td><td>$(cat status.txt)</td><td>$(cat description.txt)</td></tr>
+  <th>Test name</th>
+  <th>Test status</th>
+  <th>Description</th>
+</tr>
+<tr>
+  <td>AST Fuzzer</td>
+  <td>$(cat status.txt)</td>
+  <td>$(
+    clickhouse-local --input-format RawBLOB --output-format RawBLOB --query "SELECT encodeXMLComponent(*) FROM table" < description.txt || cat description.txt
+  )</td>
+</tr>
+<tr>
+  <td colspan="3" style="white-space: pre-wrap;">$(
+    clickhouse-local --input-format RawBLOB --output-format RawBLOB --query "SELECT encodeXMLComponent(*) FROM table" < fatal.log || cat fatal.log
+  )</td>
+</tr>
 </table>
 </body>
 </html>

docker/test/stateful/Dockerfile

@@ -17,6 +17,7 @@ ENV S3_URL="https://clickhouse-datasets.s3.amazonaws.com"
 ENV DATASETS="hits visits"
 
 RUN npm install -g azurite
+RUN npm install tslib
 
 COPY run.sh /
 CMD ["/bin/bash", "/run.sh"]

docker/test/stateless/Dockerfile

@@ -80,6 +80,7 @@ ENV MINIO_ROOT_PASSWORD="clickhouse"
 ENV EXPORT_S3_STORAGE_POLICIES=1
 
 RUN npm install -g azurite
+RUN npm install tslib
 
 COPY run.sh /
 COPY setup_minio.sh /

docker/test/stateless/setup_minio.sh

@@ -75,7 +75,7 @@ fi
 TEST_PATH=${1:-/usr/share/clickhouse-test}
 MINIO_DATA_PATH=${TEST_PATH}/queries/${QUERY_DIR}/data_minio
 
-# Iterating over globs will cause redudant FILE variale to be a path to a file, not a filename
+# Iterating over globs will cause redundant FILE variable to be a path to a file, not a filename
 # shellcheck disable=SC2045
 for FILE in $(ls "${MINIO_DATA_PATH}"); do
     echo "$FILE";

docker/test/stress/README.md

@@ -1,6 +1,6 @@
-Allow to run simple ClickHouse stress test in Docker from debian packages.
+Allows to run simple ClickHouse stress test in Docker from debian packages.
 Actually it runs multiple copies of clickhouse-test (functional tests).
-This allows to find problems like segmentation fault which cause shutdown of server.
+This allows to find problems like failed assertions and memory safety issues.
 
 Usage:
 ```

docker/test/stress/run.sh

@@ -11,31 +11,6 @@ set -x
 # core.COMM.PID-TID
 sysctl kernel.core_pattern='core.%e.%p-%P'
 
-# Thread Fuzzer allows to check more permutations of possible thread scheduling
-# and find more potential issues.
-# Temporarily disable ThreadFuzzer with tsan because of https://github.com/google/sanitizers/issues/1540
-is_tsan_build=$(clickhouse local -q "select value like '% -fsanitize=thread %' from system.build_options where name='CXX_FLAGS'")
-if [ "$is_tsan_build" -eq "0" ]; then
-    export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
-    export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
-    export THREAD_FUZZER_SLEEP_TIME_US=100000
-
-    export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
-    export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
-    export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_MIGRATE_PROBABILITY=1
-    export THREAD_FUZZER_pthread_mutex_unlock_AFTER_MIGRATE_PROBABILITY=1
-
-    export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001
-    export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
-    export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
-    export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
-    export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
-
-    export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
-    export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
-    export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
-fi
-
 
 function install_packages()
 {
@@ -54,7 +29,7 @@ function configure()
 
     # we mount tests folder from repo to /usr/share
     ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
-    ln -s /usr/share/clickhouse-test/ci/download_release_packets.py /usr/bin/download_release_packets
+    ln -s /usr/share/clickhouse-test/ci/download_release_packages.py /usr/bin/download_release_packages
     ln -s /usr/share/clickhouse-test/ci/get_previous_release_tag.py /usr/bin/get_previous_release_tag
 
     # avoid too slow startup
@@ -123,6 +98,22 @@ EOL
     <core_path>$PWD</core_path>
 </clickhouse>
 EOL
+
+    # Analyzer is not yet ready for testing
+    cat > /etc/clickhouse-server/users.d/no_analyzer.xml <<EOL
+<clickhouse>
+    <profiles>
+        <default>
+            <constraints>
+                <allow_experimental_analyzer>
+                    <readonly/>
+                </allow_experimental_analyzer>
+            </constraints>
+        </default>
+    </profiles>
+</clickhouse>
+EOL
+
 }
 
 function stop()
@@ -210,6 +201,31 @@ quit
 
 install_packages package_folder
 
+# Thread Fuzzer allows to check more permutations of possible thread scheduling
+# and find more potential issues.
+# Temporarily disable ThreadFuzzer with tsan because of https://github.com/google/sanitizers/issues/1540
+is_tsan_build=$(clickhouse local -q "select value like '% -fsanitize=thread %' from system.build_options where name='CXX_FLAGS'")
+if [ "$is_tsan_build" -eq "0" ]; then
+    export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
+    export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
+    export THREAD_FUZZER_SLEEP_TIME_US=100000
+
+    export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
+    export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
+    export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_MIGRATE_PROBABILITY=1
+    export THREAD_FUZZER_pthread_mutex_unlock_AFTER_MIGRATE_PROBABILITY=1
+
+    export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001
+    export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
+    export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
+    export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
+    export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
+
+    export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
+    export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
+    export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
+fi
+
 export ZOOKEEPER_FAULT_INJECTION=1
 configure
 
@@ -334,219 +350,228 @@ zgrep -Fa "########################################" /test_output/* > /dev/null
 zgrep -Fa " received signal " /test_output/gdb.log > /dev/null \
     && echo -e 'Found signal in gdb.log\tFAIL' >> /test_output/test_results.tsv
 
-echo -e "Backward compatibility check\n"
+if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
+    echo -e "Backward compatibility check\n"
 
-echo "Get previous release tag"
+    echo "Get previous release tag"
-previous_release_tag=$(clickhouse-client --version | grep -o "[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*" | get_previous_release_tag)
+    previous_release_tag=$(clickhouse-client --version | grep -o "[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*" | get_previous_release_tag)
-echo $previous_release_tag
+    echo $previous_release_tag
 
-echo "Clone previous release repository"
+    echo "Clone previous release repository"
-git clone https://github.com/ClickHouse/ClickHouse.git --no-tags --progress --branch=$previous_release_tag --no-recurse-submodules --depth=1 previous_release_repository
+    git clone https://github.com/ClickHouse/ClickHouse.git --no-tags --progress --branch=$previous_release_tag --no-recurse-submodules --depth=1 previous_release_repository
 
-echo "Download previous release server"
+    echo "Download clickhouse-server from the previous release"
-mkdir previous_release_package_folder
+    mkdir previous_release_package_folder
 
-echo $previous_release_tag | download_release_packets && echo -e 'Download script exit code\tOK' >> /test_output/test_results.tsv \
+    echo $previous_release_tag | download_release_packages && echo -e 'Download script exit code\tOK' >> /test_output/test_results.tsv \
-    || echo -e 'Download script failed\tFAIL' >> /test_output/test_results.tsv
+        || echo -e 'Download script failed\tFAIL' >> /test_output/test_results.tsv
 
-mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.clean.log
+    mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.clean.log
-for table in query_log trace_log
-do
-    clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.tsv.gz ||:
-done
-
-tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
-
-# Check if we cloned previous release repository successfully
-if ! [ "$(ls -A previous_release_repository/tests/queries)" ]
-then
-    echo -e "Backward compatibility check: Failed to clone previous release tests\tFAIL" >> /test_output/test_results.tsv
-elif ! [ "$(ls -A previous_release_package_folder/clickhouse-common-static_*.deb && ls -A previous_release_package_folder/clickhouse-server_*.deb)" ]
-then
-    echo -e "Backward compatibility check: Failed to download previous release packets\tFAIL" >> /test_output/test_results.tsv
-else
-    echo -e "Successfully cloned previous release tests\tOK" >> /test_output/test_results.tsv
-    echo -e "Successfully downloaded previous release packets\tOK" >> /test_output/test_results.tsv
-
-    # Uninstall current packages
-    dpkg --remove clickhouse-client
-    dpkg --remove clickhouse-server
-    dpkg --remove clickhouse-common-static-dbg
-    dpkg --remove clickhouse-common-static
-
-    rm -rf /var/lib/clickhouse/*
-
-    # Make BC check more funny by forcing Ordinary engine for system database
-    mkdir /var/lib/clickhouse/metadata
-    echo "ATTACH DATABASE system ENGINE=Ordinary" > /var/lib/clickhouse/metadata/system.sql
-
-    # Install previous release packages
-    install_packages previous_release_package_folder
-
-    # Start server from previous release
-    # Previous version may not be ready for fault injections
-    export ZOOKEEPER_FAULT_INJECTION=0
-    configure
-
-    # Avoid "Setting s3_check_objects_after_upload is neither a builtin setting..."
-    rm -f /etc/clickhouse-server/users.d/enable_blobs_check.xml ||:
-    rm -f /etc/clickhouse-server/users.d/marks.xml ||:
-
-    # Remove s3 related configs to avoid "there is no disk type `cache`"
-    rm -f /etc/clickhouse-server/config.d/storage_conf.xml ||:
-    rm -f /etc/clickhouse-server/config.d/azure_storage_conf.xml ||:
-
-    # Turn on after 22.12
-    rm -f /etc/clickhouse-server/config.d/compressed_marks_and_index.xml ||:
-    # it uses recently introduced settings which previous versions may not have
-    rm -f /etc/clickhouse-server/users.d/insert_keeper_retries.xml ||:
-
-    start
-
-    clickhouse-client --query="SELECT 'Server version: ', version()"
-
-    # Install new package before running stress test because we should use new
-    # clickhouse-client and new clickhouse-test.
-    #
-    # But we should leave old binary in /usr/bin/ and debug symbols in
-    # /usr/lib/debug/usr/bin (if any) for gdb and internal DWARF parser, so it
-    # will print sane stacktraces and also to avoid possible crashes.
-    #
-    # FIXME: those files can be extracted directly from debian package, but
-    # actually better solution will be to use different PATH instead of playing
-    # games with files from packages.
-    mv /usr/bin/clickhouse previous_release_package_folder/
-    mv /usr/lib/debug/usr/bin/clickhouse.debug previous_release_package_folder/
-    install_packages package_folder
-    mv /usr/bin/clickhouse package_folder/
-    mv /usr/lib/debug/usr/bin/clickhouse.debug package_folder/
-    mv previous_release_package_folder/clickhouse /usr/bin/
-    mv previous_release_package_folder/clickhouse.debug /usr/lib/debug/usr/bin/clickhouse.debug
-
-    mkdir tmp_stress_output
-
-    ./stress --test-cmd="/usr/bin/clickhouse-test --queries=\"previous_release_repository/tests/queries\""  --backward-compatibility-check --output-folder tmp_stress_output --global-time-limit=1200 \
-        && echo -e 'Backward compatibility check: Test script exit code\tOK' >> /test_output/test_results.tsv \
-        || echo -e 'Backward compatibility check: Test script failed\tFAIL' >> /test_output/test_results.tsv
-    rm -rf tmp_stress_output
-
-    clickhouse-client --query="SELECT 'Tables count:', count() FROM system.tables"
-
-    stop 1
-    mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.stress.log
-
-    # Start new server
-    mv package_folder/clickhouse /usr/bin/
-    mv package_folder/clickhouse.debug /usr/lib/debug/usr/bin/clickhouse.debug
-    export ZOOKEEPER_FAULT_INJECTION=1
-    configure
-    start 500
-    clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK'" >> /test_output/test_results.tsv \
-        || (echo -e 'Backward compatibility check: Server failed to start\tFAIL' >> /test_output/test_results.tsv \
-        && grep -a "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log >> /test_output/bc_check_application_errors.txt)
-
-    clickhouse-client --query="SELECT 'Server version: ', version()"
-
-    # Let the server run for a while before checking log.
-    sleep 60
-
-    stop
-    mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.clean.log
-
-    # Error messages (we should ignore some errors)
-    # FIXME https://github.com/ClickHouse/ClickHouse/issues/38643 ("Unknown index: idx.")
-    # FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 ("Cannot parse string 'Hello' as UInt64")
-    # FIXME Not sure if it's expected, but some tests from BC check may not be finished yet when we restarting server.
-    #       Let's just ignore all errors from queries ("} <Error> TCPHandler: Code:", "} <Error> executeQuery: Code:")
-    # FIXME https://github.com/ClickHouse/ClickHouse/issues/39197 ("Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'")
-    # NOTE  Incompatibility was introduced in https://github.com/ClickHouse/ClickHouse/pull/39263, it's expected
-    #       ("This engine is deprecated and is not supported in transactions", "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part")
-    # FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 - bad mutation does not indicate backward incompatibility
-    echo "Check for Error messages in server log:"
-    zgrep -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \
-               -e "Code: 236. DB::Exception: Cancelled mutating parts" \
-               -e "REPLICA_IS_ALREADY_ACTIVE" \
-               -e "REPLICA_ALREADY_EXISTS" \
-               -e "ALL_REPLICAS_LOST" \
-               -e "DDLWorker: Cannot parse DDL task query" \
-               -e "RaftInstance: failed to accept a rpc connection due to error 125" \
-               -e "UNKNOWN_DATABASE" \
-               -e "NETWORK_ERROR" \
-               -e "UNKNOWN_TABLE" \
-               -e "ZooKeeperClient" \
-               -e "KEEPER_EXCEPTION" \
-               -e "DirectoryMonitor" \
-               -e "TABLE_IS_READ_ONLY" \
-               -e "Code: 1000, e.code() = 111, Connection refused" \
-               -e "UNFINISHED" \
-               -e "NETLINK_ERROR" \
-               -e "Renaming unexpected part" \
-               -e "PART_IS_TEMPORARILY_LOCKED" \
-               -e "and a merge is impossible: we didn't find" \
-               -e "found in queue and some source parts for it was lost" \
-               -e "is lost forever." \
-               -e "Unknown index: idx." \
-               -e "Cannot parse string 'Hello' as UInt64" \
-               -e "} <Error> TCPHandler: Code:" \
-               -e "} <Error> executeQuery: Code:" \
-               -e "Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'" \
-               -e "This engine is deprecated and is not supported in transactions" \
-               -e "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part" \
-               -e "The set of parts restored in place of" \
-               -e "(ReplicatedMergeTreeAttachThread): Initialization failed. Error" \
-               -e "Code: 269. DB::Exception: Destination table is myself" \
-               -e "Coordination::Exception: Connection loss" \
-               -e "MutateFromLogEntryTask" \
-               -e "No connection to ZooKeeper, cannot get shared table ID" \
-               -e "Session expired" \
-        /var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
-        && echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
-        || echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
-
-    # Remove file bc_check_error_messages.txt if it's empty
-    [ -s /test_output/bc_check_error_messages.txt ] || rm /test_output/bc_check_error_messages.txt
-
-    # Sanitizer asserts
-    zgrep -Fa "==================" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
-    zgrep -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
-    zgrep -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \
-        && echo -e 'Backward compatibility check: Sanitizer assert (in stderr.log)\tFAIL' >> /test_output/test_results.tsv \
-        || echo -e 'Backward compatibility check: No sanitizer asserts\tOK' >> /test_output/test_results.tsv
-    rm -f /test_output/tmp
-
-    # OOM
-    zgrep -Fa " <Fatal> Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
-        && echo -e 'Backward compatibility check: OOM killer (or signal 9) in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \
-        || echo -e 'Backward compatibility check: No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
-
-    # Logical errors
-    echo "Check for Logical errors in server log:"
-    zgrep -Fa -A20 "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_logical_errors.txt \
-        && echo -e 'Backward compatibility check: Logical error thrown (see clickhouse-server.log or bc_check_logical_errors.txt)\tFAIL' >> /test_output/test_results.tsv \
-        || echo -e 'Backward compatibility check: No logical errors\tOK' >> /test_output/test_results.tsv
-
-    # Remove file bc_check_logical_errors.txt if it's empty
-    [ -s /test_output/bc_check_logical_errors.txt ] || rm /test_output/bc_check_logical_errors.txt
-
-    # Crash
-    zgrep -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
-        && echo -e 'Backward compatibility check: Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \
-        || echo -e 'Backward compatibility check: Not crashed\tOK' >> /test_output/test_results.tsv
-
-    # It also checks for crash without stacktrace (printed by watchdog)
-    echo "Check for Fatal message in server log:"
-    zgrep -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_fatal_messages.txt \
-        && echo -e 'Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
-        || echo -e 'Backward compatibility check: No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
-
-    # Remove file bc_check_fatal_messages.txt if it's empty
-    [ -s /test_output/bc_check_fatal_messages.txt ] || rm /test_output/bc_check_fatal_messages.txt
-
-    tar -chf /test_output/coordination.backward.tar /var/lib/clickhouse/coordination ||:
     for table in query_log trace_log
     do
-        clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.backward.tsv.gz ||:
+        clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.tsv.gz ||:
     done
+
+    tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
+
+    # Check if we cloned previous release repository successfully
+    if ! [ "$(ls -A previous_release_repository/tests/queries)" ]
+    then
+        echo -e "Backward compatibility check: Failed to clone previous release tests\tFAIL" >> /test_output/test_results.tsv
+    elif ! [ "$(ls -A previous_release_package_folder/clickhouse-common-static_*.deb && ls -A previous_release_package_folder/clickhouse-server_*.deb)" ]
+    then
+        echo -e "Backward compatibility check: Failed to download previous release packages\tFAIL" >> /test_output/test_results.tsv
+    else
+        echo -e "Successfully cloned previous release tests\tOK" >> /test_output/test_results.tsv
+        echo -e "Successfully downloaded previous release packages\tOK" >> /test_output/test_results.tsv
+
+        # Uninstall current packages
+        dpkg --remove clickhouse-client
+        dpkg --remove clickhouse-server
+        dpkg --remove clickhouse-common-static-dbg
+        dpkg --remove clickhouse-common-static
+
+        rm -rf /var/lib/clickhouse/*
+
+        # Make BC check more funny by forcing Ordinary engine for system database
+        mkdir /var/lib/clickhouse/metadata
+        echo "ATTACH DATABASE system ENGINE=Ordinary" > /var/lib/clickhouse/metadata/system.sql
+
+        # Install previous release packages
+        install_packages previous_release_package_folder
+
+        # Start server from previous release
+        # Previous version may not be ready for fault injections
+        export ZOOKEEPER_FAULT_INJECTION=0
+        configure
+
+        # Avoid "Setting s3_check_objects_after_upload is neither a builtin setting..."
+        rm -f /etc/clickhouse-server/users.d/enable_blobs_check.xml ||:
+        rm -f /etc/clickhouse-server/users.d/marks.xml ||:
+
+        # Remove s3 related configs to avoid "there is no disk type `cache`"
+        rm -f /etc/clickhouse-server/config.d/storage_conf.xml ||:
+        rm -f /etc/clickhouse-server/config.d/azure_storage_conf.xml ||:
+
+        # Turn on after 22.12
+        rm -f /etc/clickhouse-server/config.d/compressed_marks_and_index.xml ||:
+        # it uses recently introduced settings which previous versions may not have
+        rm -f /etc/clickhouse-server/users.d/insert_keeper_retries.xml ||:
+
+        start
+
+        clickhouse-client --query="SELECT 'Server version: ', version()"
+
+        # Install new package before running stress test because we should use new
+        # clickhouse-client and new clickhouse-test.
+        #
+        # But we should leave old binary in /usr/bin/ and debug symbols in
+        # /usr/lib/debug/usr/bin (if any) for gdb and internal DWARF parser, so it
+        # will print sane stacktraces and also to avoid possible crashes.
+        #
+        # FIXME: those files can be extracted directly from debian package, but
+        # actually better solution will be to use different PATH instead of playing
+        # games with files from packages.
+        mv /usr/bin/clickhouse previous_release_package_folder/
+        mv /usr/lib/debug/usr/bin/clickhouse.debug previous_release_package_folder/
+        install_packages package_folder
+        mv /usr/bin/clickhouse package_folder/
+        mv /usr/lib/debug/usr/bin/clickhouse.debug package_folder/
+        mv previous_release_package_folder/clickhouse /usr/bin/
+        mv previous_release_package_folder/clickhouse.debug /usr/lib/debug/usr/bin/clickhouse.debug
+
+        mkdir tmp_stress_output
+
+        ./stress --test-cmd="/usr/bin/clickhouse-test --queries=\"previous_release_repository/tests/queries\""  --backward-compatibility-check --output-folder tmp_stress_output --global-time-limit=1200 \
+            && echo -e 'Backward compatibility check: Test script exit code\tOK' >> /test_output/test_results.tsv \
+            || echo -e 'Backward compatibility check: Test script failed\tFAIL' >> /test_output/test_results.tsv
+        rm -rf tmp_stress_output
+
+        # We experienced deadlocks in this command in very rare cases. Let's debug it:
+        timeout 10m clickhouse-client --query="SELECT 'Tables count:', count() FROM system.tables" ||
+        (
+            echo "thread apply all backtrace (on select tables count)" >> /test_output/gdb.log
+            timeout 30m gdb -batch -ex 'thread apply all backtrace' -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log
+            clickhouse stop --force
+        )
+
+        stop 1
+        mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.stress.log
+
+        # Start new server
+        mv package_folder/clickhouse /usr/bin/
+        mv package_folder/clickhouse.debug /usr/lib/debug/usr/bin/clickhouse.debug
+        # Disable fault injections on start (we don't test them here, and it can lead to tons of requests in case of huge number of tables).
+        export ZOOKEEPER_FAULT_INJECTION=0
+        configure
+        start 500
+        clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK'" >> /test_output/test_results.tsv \
+            || (echo -e 'Backward compatibility check: Server failed to start\tFAIL' >> /test_output/test_results.tsv \
+            && grep -a "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log >> /test_output/bc_check_application_errors.txt)
+
+        clickhouse-client --query="SELECT 'Server version: ', version()"
+
+        # Let the server run for a while before checking log.
+        sleep 60
+
+        stop
+        mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.dirty.log
+
+        # Error messages (we should ignore some errors)
+        # FIXME https://github.com/ClickHouse/ClickHouse/issues/38643 ("Unknown index: idx.")
+        # FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 ("Cannot parse string 'Hello' as UInt64")
+        # FIXME Not sure if it's expected, but some tests from BC check may not be finished yet when we restarting server.
+        #       Let's just ignore all errors from queries ("} <Error> TCPHandler: Code:", "} <Error> executeQuery: Code:")
+        # FIXME https://github.com/ClickHouse/ClickHouse/issues/39197 ("Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'")
+        # NOTE  Incompatibility was introduced in https://github.com/ClickHouse/ClickHouse/pull/39263, it's expected
+        #       ("This engine is deprecated and is not supported in transactions", "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part")
+        # FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 - bad mutation does not indicate backward incompatibility
+        echo "Check for Error messages in server log:"
+        zgrep -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \
+                   -e "Code: 236. DB::Exception: Cancelled mutating parts" \
+                   -e "REPLICA_IS_ALREADY_ACTIVE" \
+                   -e "REPLICA_ALREADY_EXISTS" \
+                   -e "ALL_REPLICAS_LOST" \
+                   -e "DDLWorker: Cannot parse DDL task query" \
+                   -e "RaftInstance: failed to accept a rpc connection due to error 125" \
+                   -e "UNKNOWN_DATABASE" \
+                   -e "NETWORK_ERROR" \
+                   -e "UNKNOWN_TABLE" \
+                   -e "ZooKeeperClient" \
+                   -e "KEEPER_EXCEPTION" \
+                   -e "DirectoryMonitor" \
+                   -e "TABLE_IS_READ_ONLY" \
+                   -e "Code: 1000, e.code() = 111, Connection refused" \
+                   -e "UNFINISHED" \
+                   -e "NETLINK_ERROR" \
+                   -e "Renaming unexpected part" \
+                   -e "PART_IS_TEMPORARILY_LOCKED" \
+                   -e "and a merge is impossible: we didn't find" \
+                   -e "found in queue and some source parts for it was lost" \
+                   -e "is lost forever." \
+                   -e "Unknown index: idx." \
+                   -e "Cannot parse string 'Hello' as UInt64" \
+                   -e "} <Error> TCPHandler: Code:" \
+                   -e "} <Error> executeQuery: Code:" \
+                   -e "Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'" \
+                   -e "This engine is deprecated and is not supported in transactions" \
+                   -e "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part" \
+                   -e "The set of parts restored in place of" \
+                   -e "(ReplicatedMergeTreeAttachThread): Initialization failed. Error" \
+                   -e "Code: 269. DB::Exception: Destination table is myself" \
+                   -e "Coordination::Exception: Connection loss" \
+                   -e "MutateFromLogEntryTask" \
+                   -e "No connection to ZooKeeper, cannot get shared table ID" \
+                   -e "Session expired" \
+            /var/log/clickhouse-server/clickhouse-server.backward.dirty.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
+            && echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
+            || echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
+
+        # Remove file bc_check_error_messages.txt if it's empty
+        [ -s /test_output/bc_check_error_messages.txt ] || rm /test_output/bc_check_error_messages.txt
+
+        # Sanitizer asserts
+        zgrep -Fa "==================" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
+        zgrep -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
+        zgrep -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \
+            && echo -e 'Backward compatibility check: Sanitizer assert (in stderr.log)\tFAIL' >> /test_output/test_results.tsv \
+            || echo -e 'Backward compatibility check: No sanitizer asserts\tOK' >> /test_output/test_results.tsv
+        rm -f /test_output/tmp
+
+        # OOM
+        zgrep -Fa " <Fatal> Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
+            && echo -e 'Backward compatibility check: OOM killer (or signal 9) in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \
+            || echo -e 'Backward compatibility check: No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
+
+        # Logical errors
+        echo "Check for Logical errors in server log:"
+        zgrep -Fa -A20 "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_logical_errors.txt \
+            && echo -e 'Backward compatibility check: Logical error thrown (see clickhouse-server.log or bc_check_logical_errors.txt)\tFAIL' >> /test_output/test_results.tsv \
+            || echo -e 'Backward compatibility check: No logical errors\tOK' >> /test_output/test_results.tsv
+
+        # Remove file bc_check_logical_errors.txt if it's empty
+        [ -s /test_output/bc_check_logical_errors.txt ] || rm /test_output/bc_check_logical_errors.txt
+
+        # Crash
+        zgrep -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
+            && echo -e 'Backward compatibility check: Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \
+            || echo -e 'Backward compatibility check: Not crashed\tOK' >> /test_output/test_results.tsv
+
+        # It also checks for crash without stacktrace (printed by watchdog)
+        echo "Check for Fatal message in server log:"
+        zgrep -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_fatal_messages.txt \
+            && echo -e 'Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
+            || echo -e 'Backward compatibility check: No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
+
+        # Remove file bc_check_fatal_messages.txt if it's empty
+        [ -s /test_output/bc_check_fatal_messages.txt ] || rm /test_output/bc_check_fatal_messages.txt
+
+        tar -chf /test_output/coordination.backward.tar /var/lib/clickhouse/coordination ||:
+        for table in query_log trace_log
+        do
+            clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.backward.tsv.gz ||:
+        done
+    fi
 fi
 
 dmesg -T > /test_output/dmesg.log

docker/test/stress/stress

@@ -14,9 +14,6 @@ def get_options(i, backward_compatibility_check):
     if 0 < i:
         options.append("--order=random")
 
-    if i % 3 == 1:
-        options.append("--db-engine=Ordinary")
-
     if i % 3 == 2 and not backward_compatibility_check:
         options.append(
             '''--db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i)

docker/test/style/process_style_check_result.py

@@ -19,6 +19,7 @@ def process_result(result_folder):
         "typos",
         "whitespaces",
         "workflows",
+        "submodules",
         "docs spelling",
     )
 

docker/test/style/run.sh

@@ -10,7 +10,7 @@ echo "Check style" | ts
 echo "Check python formatting with black" | ts
 ./check-black -n              |& tee /test_output/black_output.txt
 echo "Check python type hinting with mypy" | ts
-./check-mypy -n              |& tee /test_output/mypy_output.txt
+./check-mypy -n               |& tee /test_output/mypy_output.txt
 echo "Check typos" | ts
 ./check-typos                 |& tee /test_output/typos_output.txt
 echo "Check docs spelling" | ts
@@ -19,6 +19,8 @@ echo "Check whitespaces" | ts
 ./check-whitespaces -n        |& tee /test_output/whitespaces_output.txt
 echo "Check workflows" | ts
 ./check-workflows             |& tee /test_output/workflows_output.txt
+echo "Check submodules" | ts
+./check-submodules            |& tee /test_output/submodules_output.txt
 echo "Check shell scripts with shellcheck" | ts
 ./shellcheck-run.sh           |& tee /test_output/shellcheck_output.txt
 /process_style_check_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv

docker/test/testflows/runner/Dockerfile

@@ -1,82 +0,0 @@
-# docker build -t clickhouse/testflows-runner .
-FROM ubuntu:20.04
-
-# ARG for quick switch to a given ubuntu mirror
-ARG apt_archive="http://archive.ubuntu.com"
-RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
-
-RUN apt-get update \
-    && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
-    ca-certificates \
-    bash \
-    btrfs-progs \
-    e2fsprogs \
-    iptables \
-    xfsprogs \
-    tar \
-    pigz \
-    wget \
-    git \
-    iproute2 \
-    cgroupfs-mount \
-    python3-pip \
-    tzdata \
-    libicu-dev \
-    bsdutils \
-    curl \
-    liblua5.1-dev \
-    luajit \
-    libssl-dev \
-    libcurl4-openssl-dev \
-    gdb \
-    && rm -rf \
-        /var/lib/apt/lists/* \
-        /var/cache/debconf \
-        /tmp/* \
-    && apt-get clean
-
-ENV TZ=Europe/Moscow
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-
-RUN pip3 install urllib3 testflows==1.7.20 docker-compose==1.29.2 docker==5.0.0 dicttoxml kazoo tzlocal==2.1 pytz python-dateutil numpy
-
-ENV DOCKER_CHANNEL stable
-ENV DOCKER_VERSION 20.10.6
-
-# Architecture of the image when BuildKit/buildx is used
-ARG TARGETARCH
-
-# Install docker
-RUN arch=${TARGETARCH:-amd64} \
-  && case $arch in \
-      amd64) rarch=x86_64 ;; \
-      arm64) rarch=aarch64 ;; \
-    esac \
-  && set -eux \
-  && if ! wget -nv -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${rarch}/docker-${DOCKER_VERSION}.tgz"; then \
-    echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${rarch}'" \
-    && exit 1; \
-  fi \
-  && tar --extract \
-    --file docker.tgz \
-    --strip-components 1 \
-    --directory /usr/local/bin/ \
-  && rm docker.tgz \
-  && dockerd --version \
-  && docker --version
-
-COPY modprobe.sh /usr/local/bin/modprobe
-COPY dockerd-entrypoint.sh /usr/local/bin/
-COPY process_testflows_result.py /usr/local/bin/
-
-RUN set -x \
-  && addgroup --system dockremap \
-    && adduser --system dockremap \
-  && adduser dockremap dockremap \
-  && echo 'dockremap:165536:65536' >> /etc/subuid \
-    && echo 'dockremap:165536:65536' >> /etc/subgid
-
-VOLUME /var/lib/docker
-EXPOSE 2375
-ENTRYPOINT ["dockerd-entrypoint.sh"]
-CMD ["sh", "-c", "python3 regression.py --no-color -o new-fails --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json; /usr/local/bin/process_testflows_result.py || echo -e 'failure\tCannot parse results' > check_status.tsv; find * -type f | grep _instances | grep clickhouse-server | xargs -n1 tar -rvf clickhouse_logs.tar; gzip -9 clickhouse_logs.tar"]

docker/test/testflows/runner/dockerd-entrypoint.sh

@@ -1,39 +0,0 @@
-#!/bin/bash
-set -e
-
-echo "Configure to use Yandex dockerhub-proxy"
-mkdir -p /etc/docker/
-cat > /etc/docker/daemon.json << EOF
-{
-    "insecure-registries" : ["dockerhub-proxy.dockerhub-proxy-zone:5000"],
-    "registry-mirrors" : ["http://dockerhub-proxy.dockerhub-proxy-zone:5000"]
-}
-EOF
-
-# In case of test hung it is convenient to use pytest --pdb to debug it,
-# and on hung you can simply press Ctrl-C and it will spawn a python pdb,
-# but on SIGINT dockerd will exit, so ignore it to preserve the daemon.
-trap '' INT
-dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 &>/var/log/somefile &
-
-set +e
-reties=0
-while true; do
-    docker info &>/dev/null && break
-    reties=$((reties+1))
-    if [[ $reties -ge 100 ]]; then # 10 sec max
-        echo "Can't start docker daemon, timeout exceeded." >&2
-        exit 1;
-    fi
-    sleep 0.1
-done
-set -e
-
-echo "Start tests"
-export CLICKHOUSE_TESTS_SERVER_BIN_PATH=/clickhouse
-export CLICKHOUSE_TESTS_CLIENT_BIN_PATH=/clickhouse
-export CLICKHOUSE_TESTS_BASE_CONFIG_DIR=/clickhouse-config
-export CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH=/clickhouse-odbc-bridge
-
-cd /ClickHouse/tests/testflows
-exec "$@"

docker/test/testflows/runner/modprobe.sh

@@ -1,20 +0,0 @@
-#!/bin/sh
-set -eu
-
-# "modprobe" without modprobe
-# https://twitter.com/lucabruno/status/902934379835662336
-
-# this isn't 100% fool-proof, but it'll have a much higher success rate than simply using the "real" modprobe
-
-# Docker often uses "modprobe -va foo bar baz"
-# so we ignore modules that start with "-"
-for module; do
-    if [ "${module#-}" = "$module" ]; then
-        ip link show "$module" || true
-        lsmod | grep "$module" || true
-    fi
-done
-
-# remove /usr/local/... from PATH so we can exec the real modprobe as a last resort
-export PATH='/usr/sbin:/usr/bin:/sbin:/bin'
-exec modprobe "$@"

docker/test/testflows/runner/process_testflows_result.py

@@ -1,71 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import logging
-import argparse
-import csv
-import json
-
-
-def process_result(result_folder):
-    json_path = os.path.join(result_folder, "results.json")
-    if not os.path.exists(json_path):
-        return "success", "No testflows in branch", None, []
-
-    test_binary_log = os.path.join(result_folder, "test.log")
-    with open(json_path) as source:
-        results = json.loads(source.read())
-
-    total_tests = 0
-    total_ok = 0
-    total_fail = 0
-    total_other = 0
-    test_results = []
-    for test in results["tests"]:
-        test_name = test["test"]["test_name"]
-        test_result = test["result"]["result_type"].upper()
-        test_time = str(test["result"]["message_rtime"])
-        total_tests += 1
-        if test_result == "OK":
-            total_ok += 1
-        elif test_result == "FAIL" or test_result == "ERROR":
-            total_fail += 1
-        else:
-            total_other += 1
-
-        test_results.append((test_name, test_result, test_time))
-    if total_fail != 0:
-        status = "failure"
-    else:
-        status = "success"
-
-    description = "failed: {}, passed: {}, other: {}".format(
-        total_fail, total_ok, total_other
-    )
-    return status, description, test_results, [json_path, test_binary_log]
-
-
-def write_results(results_file, status_file, results, status):
-    with open(results_file, "w") as f:
-        out = csv.writer(f, delimiter="\t")
-        out.writerows(results)
-    with open(status_file, "w") as f:
-        out = csv.writer(f, delimiter="\t")
-        out.writerow(status)
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
-    parser = argparse.ArgumentParser(
-        description="ClickHouse script for parsing results of Testflows tests"
-    )
-    parser.add_argument("--in-results-dir", default="./")
-    parser.add_argument("--out-results-file", default="./test_results.tsv")
-    parser.add_argument("--out-status-file", default="./check_status.tsv")
-    args = parser.parse_args()
-
-    state, description, test_results, logs = process_result(args.in_results_dir)
-    logging.info("Result parsed")
-    status = (state, description)
-    write_results(args.out_results_file, args.out_status_file, test_results, status)
-    logging.info("Result written")

docs/_includes/install/universal.sh

@@ -9,14 +9,22 @@ if [ "${OS}" = "Linux" ]
 then
     if [ "${ARCH}" = "x86_64" -o "${ARCH}" = "amd64" ]
     then
-        DIR="amd64"
+        # Require at least x86-64 + SSE4.2 (introduced in 2006). On older hardware fall back to plain x86-64 (introduced in 1999) which
+        # guarantees at least SSE2. The caveat is that plain x86-64 builds are much less tested than SSE 4.2 builds.
+        HAS_SSE42=$(grep sse4_2 /proc/cpuinfo)
+        if [ "${HAS_SSE42}" ]
+        then
+            DIR="amd64"
+        else
+            DIR="amd64compat"
+        fi
     elif [ "${ARCH}" = "aarch64" -o "${ARCH}" = "arm64" ]
     then
         # If the system has >=ARMv8.2 (https://en.wikipedia.org/wiki/AArch64), choose the corresponding build, else fall back to a v8.0
         # compat build. Unfortunately, the ARM ISA level cannot be read directly, we need to guess from the "features" in /proc/cpuinfo.
         # Also, the flags in /proc/cpuinfo are named differently than the flags passed to the compiler (cmake/cpu_features.cmake).
-        ARMV82=$(grep -m 1 'Features' /proc/cpuinfo | awk '/asimd/ && /sha1/ && /aes/ && /atomics/ && /lrcpc/')
+        HAS_ARMV82=$(grep -m 1 'Features' /proc/cpuinfo | awk '/asimd/ && /sha1/ && /aes/ && /atomics/ && /lrcpc/')
-        if [ "${ARMV82}" ]
+        if [ "${HAS_ARMV82}" ]
         then
             DIR="aarch64"
         else

docs/en/development/build-cross-osx.md

@@ -1,15 +1,15 @@
 ---
 slug: /en/development/build-cross-osx
 sidebar_position: 66
-title: How to Build ClickHouse on Linux for Mac OS X 
+title: How to Build ClickHouse on Linux for macOS
-sidebar_label: Build on Linux for Mac OS X
+sidebar_label: Build on Linux for macOS
 ---
 
 
 This is for the case when you have a Linux machine and want to use it to build `clickhouse` binary that will run on OS X. 
-This is intended for continuous integration checks that run on Linux servers. If you want to build ClickHouse directly on Mac OS X, then proceed with [another instruction](../development/build-osx.md).
+This is intended for continuous integration checks that run on Linux servers. If you want to build ClickHouse directly on macOS, then proceed with [another instruction](../development/build-osx.md).
 
-The cross-build for Mac OS X is based on the [Build instructions](../development/build.md), follow them first.
+The cross-build for macOS is based on the [Build instructions](../development/build.md), follow them first.
 
 ## Install Clang-14
 

docs/en/development/build-osx.md

@@ -1,9 +1,9 @@
 ---
 slug: /en/development/build-osx
 sidebar_position: 65
-sidebar_label: Build on Mac OS X
+sidebar_label: Build on macOS
-title: How to Build ClickHouse on Mac OS X
+title: How to Build ClickHouse on macOS
-description: How to build ClickHouse on Mac OS X
+description: How to build ClickHouse on macOS
 ---
 
 :::info You don't have to build ClickHouse yourself!

docs/en/development/developer-instruction.md

@@ -7,7 +7,7 @@ description: Prerequisites and an overview of how to build ClickHouse
 
 # Getting Started Guide for Building ClickHouse
 
-The building of ClickHouse is supported on Linux, FreeBSD and Mac OS X.
+The building of ClickHouse is supported on Linux, FreeBSD and macOS.
 
 If you use Windows, you need to create a virtual machine with Ubuntu. To start working with a virtual machine please install VirtualBox. You can download Ubuntu from the website: https://www.ubuntu.com/#download. Please create a virtual machine from the downloaded image (you should reserve at least 4GB of RAM for it). To run a command-line terminal in Ubuntu, please locate a program containing the word “terminal” in its name (gnome-terminal, konsole etc.) or just press Ctrl+Alt+T.
 
@@ -194,7 +194,7 @@ In this case, ClickHouse will use config files located in the current directory.
 
 To connect to ClickHouse with clickhouse-client in another terminal navigate to `ClickHouse/build/programs/` and run `./clickhouse client`.
 
-If you get `Connection refused` message on Mac OS X or FreeBSD, try specifying host address 127.0.0.1:
+If you get `Connection refused` message on macOS or FreeBSD, try specifying host address 127.0.0.1:
 
     clickhouse client --host 127.0.0.1
 
@@ -213,7 +213,7 @@ You can also run your custom-built ClickHouse binary with the config file from t
 
 ## IDE (Integrated Development Environment) {#ide-integrated-development-environment}
 
-If you do not know which IDE to use, we recommend that you use CLion. CLion is commercial software, but it offers 30 days free trial period. It is also free of charge for students. CLion can be used both on Linux and on Mac OS X.
+If you do not know which IDE to use, we recommend that you use CLion. CLion is commercial software, but it offers 30 days free trial period. It is also free of charge for students. CLion can be used both on Linux and on macOS.
 
 KDevelop and QTCreator are other great alternatives of an IDE for developing ClickHouse. KDevelop comes in as a very handy IDE although unstable. If KDevelop crashes after a while upon opening project, you should click “Stop All” button as soon as it has opened the list of project’s files. After doing so KDevelop should be fine to work with.
 

docs/en/development/tests.md

@@ -139,7 +139,7 @@ If the system clickhouse-server is already running and you do not want to stop i
 Build tests allow to check that build is not broken on various alternative configurations and on some foreign systems. These tests are automated as well.
 
 Examples:
--   cross-compile for Darwin x86_64 (Mac OS X)
+-   cross-compile for Darwin x86_64 (macOS)
 -   cross-compile for FreeBSD x86_64
 -   cross-compile for Linux AArch64
 -   build on Ubuntu with libraries from system packages (discouraged)

docs/en/getting-started/install.md

@@ -6,10 +6,11 @@ slug: /en/install
 
 # Installing ClickHouse
 
-You have two options for getting up and running with ClickHouse:
+You have three options for getting up and running with ClickHouse:
 
-- **[ClickHouse Cloud](https://clickhouse.com/cloud/):** the official ClickHouse as a service, - built by, maintained, and supported by the creators of ClickHouse
+- **[ClickHouse Cloud](https://clickhouse.com/cloud/):** The official ClickHouse as a service, - built by, maintained and supported by the creators of ClickHouse
-- **[Self-managed ClickHouse](#self-managed-install):** ClickHouse can run on any Linux, FreeBSD, or Mac OS X with x86_64, AArch64, or PowerPC64LE CPU architecture
+- **[Self-managed ClickHouse](#self-managed-install):** ClickHouse can run on any Linux, FreeBSD, or macOS with x86-64, ARM, or PowerPC64LE CPU architecture
+- **[Docker Image](https://hub.docker.com/r/clickhouse/clickhouse-server/):** Read the guide with the official image in Docker Hub
 
 ## ClickHouse Cloud
 
@@ -22,73 +23,49 @@ The quickest and easiest way to get up and running with ClickHouse is to create
 
 Once your Cloud service is provisioned, you will be able to [connect to it](/docs/en/integrations/connect-a-client.md) and start [inserting data](/docs/en/integrations/data-ingestion.md).
 
-:::note
-The [Quick Start](/docs/en/quick-start.mdx) walks through the steps to get a ClickHouse Cloud service up and running, connecting to it, and inserting data.
-:::
-
-## Self-Managed Requirements
-
-### CPU Architecture
-
-ClickHouse can run on any Linux, FreeBSD, or Mac OS X with x86_64, AArch64, or PowerPC64LE CPU architecture.
-
-Official pre-built binaries are typically compiled for x86_64 and leverage SSE 4.2 instruction set, so unless otherwise stated usage of CPU that supports it becomes an additional system requirement. Here’s the command to check if current CPU has support for SSE 4.2:
-
-``` bash
-$ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported"
-```
-
-To run ClickHouse on processors that do not support SSE 4.2 or have AArch64 or PowerPC64LE architecture, you should [build ClickHouse from sources](#from-sources) with proper configuration adjustments.
-
-ClickHouse implements parallel data processing and uses all the hardware resources available. When choosing a processor, take into account that ClickHouse works more efficiently at configurations with a large number of cores but a lower clock rate than at configurations with fewer cores and a higher clock rate. For example, 16 cores with 2600 MHz is preferable to 8 cores with 3600 MHz.
-
-It is recommended to use **Turbo Boost** and **hyper-threading** technologies. It significantly improves performance with a typical workload.
-
-### RAM {#ram}
-
-We recommend using a minimum of 4GB of RAM to perform non-trivial queries. The ClickHouse server can run with a much smaller amount of RAM, but it requires memory for processing queries.
-
-The required volume of RAM depends on:
-
--   The complexity of queries.
--   The amount of data that is processed in queries.
-
-To calculate the required volume of RAM, you should estimate the size of temporary data for [GROUP BY](/docs/en/sql-reference/statements/select/group-by.md#select-group-by-clause), [DISTINCT](/docs/en/sql-reference/statements/select/distinct.md#select-distinct), [JOIN](/docs/en/sql-reference/statements/select/join.md#select-join) and other operations you use.
-
-ClickHouse can use external memory for temporary data. See [GROUP BY in External Memory](/docs/en/sql-reference/statements/select/group-by.md#select-group-by-in-external-memory) for details.
-
-### Swap File {#swap-file}
-
-Disable the swap file for production environments.
-
-### Storage Subsystem {#storage-subsystem}
-
-You need to have 2GB of free disk space to install ClickHouse.
-
-The volume of storage required for your data should be calculated separately. Assessment should include:
-
--   Estimation of the data volume.
-
-    You can take a sample of the data and get the average size of a row from it. Then multiply the value by the number of rows you plan to store.
-
--   The data compression coefficient.
-
-    To estimate the data compression coefficient, load a sample of your data into ClickHouse, and compare the actual size of the data with the size of the table stored. For example, clickstream data is usually compressed by 6-10 times.
-
-To calculate the final volume of data to be stored, apply the compression coefficient to the estimated data volume. If you plan to store data in several replicas, then multiply the estimated volume by the number of replicas.
-
-### Network {#network}
-
-If possible, use networks of 10G or higher class.
-
-The network bandwidth is critical for processing distributed queries with a large amount of intermediate data. Besides, network speed affects replication processes.
-
-### Software {#software}
-
-ClickHouse is developed primarily for the Linux family of operating systems. The recommended Linux distribution is Ubuntu. The `tzdata` package should be installed in the system.
 
 ## Self-Managed Install
 
+1. The simplest way to download ClickHouse locally is to run the following command. If your operating system is supported, an appropriate ClickHouse binary will be downloaded and made runnable:
+  ```bash
+  curl https://clickhouse.com/ | sh
+  ```
+
+1. Run the `install` command, which defines a collection of useful symlinks along with the files and folders used by ClickHouse - all of which you can see in the output of the install script:
+  ```bash
+  sudo ./clickhouse install
+  ```
+
+1. At the end of the install script, you are prompted for a password for the `default` user. Feel free to enter a password, or you can optionally leave it blank:
+  ```response
+  Creating log directory /var/log/clickhouse-server.
+  Creating data directory /var/lib/clickhouse.
+  Creating pid directory /var/run/clickhouse-server.
+   chown -R clickhouse:clickhouse '/var/log/clickhouse-server'
+   chown -R clickhouse:clickhouse '/var/run/clickhouse-server'
+   chown  clickhouse:clickhouse '/var/lib/clickhouse'
+  Enter password for default user:
+  ```
+  You should see the following output:
+  ```response
+   ClickHouse has been successfully installed.
+
+   Start clickhouse-server with:
+    sudo clickhouse start
+
+   Start clickhouse-client with:
+    clickhouse-client
+  ```
+
+1. Run the following command to start the ClickHouse server:
+  ```bash
+  sudo clickhouse start
+  ```
+
+:::tip
+The [Quick Start](/docs/en/quick-start.mdx/#step-1-get-clickhouse) walks through the steps to download and run ClickHouse, connect to it, and insert data.
+:::
+
 ## Available Installation Options {#available-installation-options}
 
 ### From DEB Packages {#install-from-deb-packages}
@@ -278,50 +255,16 @@ For production environments, it’s recommended to use the latest `stable`-versi
 
 To run ClickHouse inside Docker follow the guide on [Docker Hub](https://hub.docker.com/r/clickhouse/clickhouse-server/). Those images use official `deb` packages inside.
 
-### Single Binary {#from-single-binary}
-
-You can install ClickHouse on Linux using a single portable binary from the latest commit of the `master` branch: [https://builds.clickhouse.com/master/amd64/clickhouse].
-
-``` bash
-curl -O 'https://builds.clickhouse.com/master/amd64/clickhouse' && chmod a+x clickhouse
-sudo ./clickhouse install
-```
-
-### From Precompiled Binaries for Non-Standard Environments {#from-binaries-non-linux}
-
-For non-Linux operating systems and for AArch64 CPU architecture, ClickHouse builds are provided as a cross-compiled binary from the latest commit of the `master` branch (with a few hours delay).
-
--   [MacOS x86_64](https://builds.clickhouse.com/master/macos/clickhouse)
-     ```bash
-     curl -O 'https://builds.clickhouse.com/master/macos/clickhouse' && chmod a+x ./clickhouse
-     ```
--   [MacOS Aarch64 (Apple Silicon)](https://builds.clickhouse.com/master/macos-aarch64/clickhouse)
-    ```bash
-    curl -O 'https://builds.clickhouse.com/master/macos-aarch64/clickhouse' && chmod a+x ./clickhouse
-    ```
--   [FreeBSD x86_64](https://builds.clickhouse.com/master/freebsd/clickhouse)
-    ```bash
-    curl -O 'https://builds.clickhouse.com/master/freebsd/clickhouse' && chmod a+x ./clickhouse
-    ```
--   [Linux AArch64](https://builds.clickhouse.com/master/aarch64/clickhouse)
-    ```bash
-    curl -O 'https://builds.clickhouse.com/master/aarch64/clickhouse' && chmod a+x ./clickhouse
-    ```
-
-Run `sudo ./clickhouse install` to install ClickHouse system-wide (also with needed configuration files, configuring users etc.). Then run `sudo clickhouse start` commands to start the clickhouse-server and `clickhouse-client` to connect to it.
-
-Use the `clickhouse client` to connect to the server, or `clickhouse local` to process local data.
-
 ### From Sources {#from-sources}
 
-To manually compile ClickHouse, follow the instructions for [Linux](/docs/en/development/build.md) or [Mac OS X](/docs/en/development/build-osx.md).
+To manually compile ClickHouse, follow the instructions for [Linux](/docs/en/development/build.md) or [macOS](/docs/en/development/build-osx.md).
 
-You can compile packages and install them or use programs without installing packages. Also by building manually you can disable SSE 4.2 requirement or build for AArch64 CPUs.
+You can compile packages and install them or use programs without installing packages.
 
-      Client: programs/clickhouse-client
+      Client: <build_directory>/programs/clickhouse-client
-      Server: programs/clickhouse-server
+      Server: <build_directory>/programs/clickhouse-server
 
-You’ll need to create a data and metadata folders and `chown` them for the desired user. Their paths can be changed in server config (src/programs/server/config.xml), by default they are:
+You’ll need to create data and metadata folders manually and `chown` them for the desired user. Their paths can be changed in server config (src/programs/server/config.xml), by default they are:
 
       /var/lib/clickhouse/data/default/
       /var/lib/clickhouse/metadata/default/
@@ -406,3 +349,42 @@ SELECT 1
 **Congratulations, the system works!**
 
 To continue experimenting, you can download one of the test data sets or go through [tutorial](/docs/en/tutorial.md).
+
+## Recommendations for Self-Managed ClickHouse
+
+ClickHouse can run on any Linux, FreeBSD, or macOS with x86-64, ARM, or PowerPC64LE CPU architecture.
+
+ClickHouse uses all hardware resources available to process data.
+
+ClickHouse tends to work more efficiently with a large number of cores at a lower clock rate than with fewer cores at a higher clock rate.
+
+We recommend using a minimum of 4GB of RAM to perform non-trivial queries. The ClickHouse server can run with a much smaller amount of RAM, but queries will then frequently abort.
+
+The required volume of RAM generally depends on:
+
+-   The complexity of queries.
+-   The amount of data that is processed in queries.
+
+To calculate the required volume of RAM, you may estimate the size of temporary data for [GROUP BY](/docs/en/sql-reference/statements/select/group-by.md#select-group-by-clause), [DISTINCT](/docs/en/sql-reference/statements/select/distinct.md#select-distinct), [JOIN](/docs/en/sql-reference/statements/select/join.md#select-join) and other operations you use.
+
+To reduce memory consumption, ClickHouse can swap temporary data to external storage. See [GROUP BY in External Memory](/docs/en/sql-reference/statements/select/group-by.md#select-group-by-in-external-memory) for details.
+
+We recommend to disable the operating system's swap file in production environments.
+
+The ClickHouse binary requires at least 2.5 GB of disk space for installation.
+
+The volume of storage required for your data may be calculated separately based on
+
+-   an estimation of the data volume.
+
+    You can take a sample of the data and get the average size of a row from it. Then multiply the value by the number of rows you plan to store.
+
+-   The data compression coefficient.
+
+    To estimate the data compression coefficient, load a sample of your data into ClickHouse, and compare the actual size of the data with the size of the table stored. For example, clickstream data is usually compressed by 6-10 times.
+
+To calculate the final volume of data to be stored, apply the compression coefficient to the estimated data volume. If you plan to store data in several replicas, then multiply the estimated volume by the number of replicas.
+
+For distributed ClickHouse deployments (clustering), we recommend at least 10G class network connectivity.
+
+Network bandwidth is critical for processing distributed queries with a large amount of intermediate data. Besides, network speed affects replication processes.

docs/en/interfaces/cli.md

@@ -2,11 +2,10 @@
 slug: /en/interfaces/cli
 sidebar_position: 17
 sidebar_label: Command-Line Client
+title: Command-Line Client
 ---
 import ConnectionDetails from '@site/docs/en/_snippets/_gather_your_details_native.md';
 
-# Command-line Client
-
 ## clickhouse-client
 
 ClickHouse provides a native command-line client: `clickhouse-client`. The client supports command-line options and configuration files. For more information, see [Configuring](#interfaces_cli_configuration).

docs/en/operations/server-configuration-parameters/settings.md

@@ -890,7 +890,7 @@ The maximum number of open files.
 
 By default: `maximum`.
 
-We recommend using this option in Mac OS X since the `getrlimit()` function returns an incorrect value.
+We recommend using this option in macOS since the `getrlimit()` function returns an incorrect value.
 
 **Example**
 

docs/en/operations/settings/settings.md

@@ -3531,13 +3531,45 @@ Default value: 2.
 
 ## compatibility {#compatibility}
 
-This setting changes other settings according to provided ClickHouse version.
+The `compatibility` setting causes ClickHouse to use the default settings of a previous version of ClickHouse, where the previous version is provided as the setting.
-If a behaviour in ClickHouse was changed by using a different default value for some setting, this compatibility setting allows you to use default values from previous versions for all the settings that were not set by the user.
 
-This setting takes ClickHouse version number as a string, like `21.3`, `21.8`. Empty value means that this setting is disabled.
+If settings are set to non-default values, then those settings are honored (only settings that have not been modified are affected by the `compatibility` setting).
+
+This setting takes a ClickHouse version number as a string, like `22.3`, `22.8`. An empty value means that this setting is disabled.
 
 Disabled by default.
 
+:::note
+In ClickHouse Cloud the compatibility setting must be set by ClickHouse Cloud support.  Please [open a case](https://clickhouse.cloud/support) to have it set.
+:::
+
+## allow_settings_after_format_in_insert {#allow_settings_after_format_in_insert}
+
+Control whether `SETTINGS` after `FORMAT` in `INSERT` queries is allowed or not. It is not recommended to use this, since this may interpret part of `SETTINGS` as values.
+
+Example:
+
+```sql
+INSERT INTO FUNCTION null('foo String') SETTINGS max_threads=1 VALUES ('bar');
+```
+
+But the following query will work only with `allow_settings_after_format_in_insert`:
+
+```sql
+SET allow_settings_after_format_in_insert=1;
+INSERT INTO FUNCTION null('foo String') VALUES ('bar') SETTINGS max_threads=1;
+```
+
+Possible values:
+
+-   0 — Disallow.
+-   1 — Allow.
+
+Default value: `0`.
+
+!!! note "Warning"
+    Use this setting only for backward compatibility if your use cases depend on old syntax.
+
 # Format settings {#format-settings}
 
 ## input_format_skip_unknown_fields {#input_format_skip_unknown_fields}
@@ -3672,6 +3704,13 @@ y	Nullable(String)
 z	IPv4
 ```
 
+## schema_inference_make_columns_nullable {#schema_inference_make_columns_nullable}
+
+Controls making inferred types `Nullable` in schema inference for formats without information about nullability.
+If the setting is enabled, the inferred type will be `Nullable` only if column contains `NULL` in a sample that is parsed during schema inference.
+
+Default value: `true`.
+
 ## input_format_try_infer_integers {#input_format_try_infer_integers}
 
 If enabled, ClickHouse will try to infer integers instead of floats in schema inference for text formats. If all numbers in the column from input data are integers, the result type will be `Int64`, if at least one number is float, the result type will be `Float64`.

docs/en/operations/system-tables/schema_inference_cache.md

@@ -0,0 +1,70 @@
+---
+slug: /en/operations/system-tables/schema_inference_cache
+---
+# Schema inference cache
+
+Contains information about all cached file schemas.
+
+Columns:
+- `storage` ([String](/docs/en/sql-reference/data-types/string.md)) — Storage name: File, URL, S3 or HDFS.
+- `source` ([String](/docs/en/sql-reference/data-types/string.md)) — File source.
+- `format` ([String](/docs/en/sql-reference/data-types/string.md)) — Format name.
+- `additional_format_info` ([String](/docs/en/sql-reference/data-types/string.md)) - Additional information required to identify the schema. For example, format specific settings.
+- `registration_time` ([DateTime](/docs/en/sql-reference/data-types/datetime.md)) — Timestamp when schema was added in cache.
+- `schema` ([String](/docs/en/sql-reference/data-types/string.md)) - Cached schema.
+
+**Example**
+
+Let's say we have a file `data.jsonl` with this content:
+```json
+{"id" :  1, "age" :  25, "name" :  "Josh", "hobbies" :  ["football", "cooking", "music"]}
+{"id" :  2, "age" :  19, "name" :  "Alan", "hobbies" :  ["tennis", "art"]}
+{"id" :  3, "age" :  32, "name" :  "Lana", "hobbies" :  ["fitness", "reading", "shopping"]}
+{"id" :  4, "age" :  47, "name" :  "Brayan", "hobbies" :  ["movies", "skydiving"]}
+```
+
+:::tip
+Place `data.jsonl` in the `user_files_path` directory.  You can find this by looking
+in your ClickHouse configuration files. The default is:
+```
+<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+```
+:::
+
+Open `clickhouse-client` and run the `DESCRIBE` query:
+
+```sql
+DESCRIBE file('data.jsonl') SETTINGS input_format_try_infer_integers=0;
+```
+
+```response
+┌─name────┬─type────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
+│ id      │ Nullable(Float64)       │              │                    │         │                  │                │
+│ age     │ Nullable(Float64)       │              │                    │         │                  │                │
+│ name    │ Nullable(String)        │              │                    │         │                  │                │
+│ hobbies │ Array(Nullable(String)) │              │                    │         │                  │                │
+└─────────┴─────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
+```
+
+Let's see the content of the `system.schema_inference_cache` table:
+
+```sql
+SELECT *
+FROM system.schema_inference_cache
+FORMAT Vertical
+```
+```response
+Row 1:
+──────
+storage:                File
+source:                 /home/droscigno/user_files/data.jsonl
+format:                 JSONEachRow
+additional_format_info: schema_inference_hints=, max_rows_to_read_for_schema_inference=25000, schema_inference_make_columns_nullable=true, try_infer_integers=false, try_infer_dates=true, try_infer_datetimes=true, try_infer_numbers_from_strings=true, read_bools_as_numbers=true, try_infer_objects=false
+registration_time:      2022-12-29 17:49:52
+schema:                 id Nullable(Float64), age Nullable(Float64), name Nullable(String), hobbies Array(Nullable(String))
+```
+
+
+**See also**
+-   [Automatic schema inference from input data](/docs/en/interfaces/schema-inference.md)
+

docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md

@@ -825,6 +825,23 @@ Setting fields:
 The `table` or `where` fields cannot be used together with the `query` field. And either one of the `table` or `query` fields must be declared.
 :::
 
+## Null
+
+A special source that can be used to create dummy (empty) dictionaries. Such dictionaries can useful for tests or with setups with separated data and query nodes at nodes with Distributed tables.
+
+``` sql
+CREATE DICTIONARY null_dict (
+    id              UInt64,
+    val             UInt8,
+    default_val     UInt8 DEFAULT 123,
+    nullable_val    Nullable(UInt8)
+)
+PRIMARY KEY id
+SOURCE(NULL())
+LAYOUT(FLAT())
+LIFETIME(0);
+```
+
 ## Related Content
 
-- [Using dictionaries to accelerate queries](https://clickhouse.com/blog/faster-queries-dictionaries-clickhouse)
+- [Using dictionaries to accelerate queries](https://clickhouse.com/blog/faster-queries-dictionaries-clickhouse)

docs/en/sql-reference/functions/date-time-functions.md

@@ -1104,6 +1104,7 @@ Using replacement fields, you can define a pattern for the resulting string. “
 | %d       | day of the month, zero-padded (01-31)                   | 02         |
 | %D       | Short MM/DD/YY date, equivalent to %m/%d/%y             | 01/02/18   |
 | %e       | day of the month, space-padded ( 1-31)                  |   2   |
+| %f       | fractional second from the fractional part of DateTime64 | 1234560   |
 | %F       | short YYYY-MM-DD date, equivalent to %Y-%m-%d           | 2018-01-02 |
 | %G       | four-digit year format for ISO week number, calculated from the week-based year [defined by the ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Week_dates) standard, normally useful only with %V  | 2018         |
 | %g       | two-digit year format, aligned to ISO 8601, abbreviated from four-digit notation                                | 18       |
@@ -1143,6 +1144,20 @@ Result:
 └────────────────────────────────────────────┘
 ```
 
+Query:
+
+``` sql
+SELECT formatDateTime(toDateTime64('2010-01-04 12:34:56.123456', 7), '%f')
+```
+
+Result:
+
+```
+┌─formatDateTime(toDateTime64('2010-01-04 12:34:56.123456', 7), '%f')─┐
+│ 1234560                                                             │
+└─────────────────────────────────────────────────────────────────────┘
+```
+
 ## dateName
 
 Returns specified part of date.

docs/en/sql-reference/functions/hash-functions.md

@@ -595,9 +595,9 @@ SELECT xxHash64('')
 
 **Returned value**
 
-A `Uint32` or `Uint64` data type hash value.
+A `UInt32` or `UInt64` data type hash value.
 
-Type: `xxHash`.
+Type: `UInt32` for `xxHash32` and `UInt64` for `xxHash64`.
 
 **Example**
 

docs/en/sql-reference/functions/random-functions.md

@@ -68,6 +68,483 @@ Result:
 └────────────┴────────────┴──────────────┴────────────────┴─────────────────┴──────────────────────┘
 ```
 
+# Functions for Generating Random Numbers based on Distributions
+
+:::note    
+These functions are available starting from 22.10.
+:::
+
+
+
+## randUniform
+
+Return random number based on [continuous uniform distribution](https://en.wikipedia.org/wiki/Continuous_uniform_distribution) in a specified range from `min` to `max`.
+
+**Syntax**
+
+``` sql
+randUniform(min, max)
+```
+
+**Arguments**
+
+- `min` - `Float64` - min value of the range,
+- `max` - `Float64` - max value of the range.
+
+**Returned value**
+
+- Pseudo-random number.
+
+Type: [Float64](/docs/en/sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT randUniform(5.5, 10) FROM numbers(5)
+```
+
+Result:
+
+``` text
+┌─randUniform(5.5, 10)─┐
+│    8.094978491443102 │
+│   7.3181248914450885 │
+│    7.177741903868262 │
+│    6.483347380953762 │
+│    6.122286382885112 │
+└──────────────────────┘
+```
+
+
+
+## randNormal
+
+Return random number based on [normal distribution](https://en.wikipedia.org/wiki/Normal_distribution).
+
+**Syntax**
+
+``` sql
+randNormal(meam, variance)
+```
+
+**Arguments**
+
+- `meam` - `Float64` mean value of distribution,
+- `variance` - `Float64` - [variance](https://en.wikipedia.org/wiki/Variance).
+
+**Returned value**
+
+- Pseudo-random number.
+
+Type: [Float64](/docs/en/sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT randNormal(10, 2) FROM numbers(5)
+```
+
+Result:
+
+``` text
+┌──randNormal(10, 2)─┐
+│ 13.389228911709653 │
+│  8.622949707401295 │
+│ 10.801887062682981 │
+│ 4.5220192605895315 │
+│ 10.901239123982567 │
+└────────────────────┘
+```
+
+
+
+## randLogNormal
+
+Return random number based on [log-normal distribution](https://en.wikipedia.org/wiki/Log-normal_distribution).
+
+**Syntax**
+
+``` sql
+randLogNormal(meam, variance)
+```
+
+**Arguments**
+
+- `meam` - `Float64` mean value of distribution,
+- `variance` - `Float64` - [variance](https://en.wikipedia.org/wiki/Variance).
+
+**Returned value**
+
+- Pseudo-random number.
+
+Type: [Float64](/docs/en/sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT randLogNormal(100, 5) FROM numbers(5)
+```
+
+Result:
+
+``` text
+┌─randLogNormal(100, 5)─┐
+│  1.295699673937363e48 │
+│  9.719869109186684e39 │
+│  6.110868203189557e42 │
+│  9.912675872925529e39 │
+│ 2.3564708490552458e42 │
+└───────────────────────┘
+```
+
+
+
+## randBinomial
+
+Return random number based on [binomial distribution](https://en.wikipedia.org/wiki/Binomial_distribution).
+
+**Syntax**
+
+``` sql
+randBinomial(experiments, probability)
+```
+
+**Arguments**
+
+- `experiments` - `UInt64` number of experiments,
+- `probability` - `Float64` - probability of success in each experiment (values in `0...1` range only).
+
+**Returned value**
+
+- Pseudo-random number.
+
+Type: [UInt64](/docs/en/sql-reference/data-types/int-uint.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT randBinomial(100, .75) FROM numbers(5)
+```
+
+Result:
+
+``` text
+┌─randBinomial(100, 0.75)─┐
+│                      74 │
+│                      78 │
+│                      76 │
+│                      77 │
+│                      80 │
+└─────────────────────────┘
+```
+
+
+
+## randNegativeBinomial
+
+Return random number based on [negative binomial distribution](https://en.wikipedia.org/wiki/Negative_binomial_distribution).
+
+**Syntax**
+
+``` sql
+randNegativeBinomial(experiments, probability)
+```
+
+**Arguments**
+
+- `experiments` - `UInt64` number of experiments,
+- `probability` - `Float64` - probability of failure in each experiment (values in `0...1` range only).
+
+**Returned value**
+
+- Pseudo-random number.
+
+Type: [UInt64](/docs/en/sql-reference/data-types/int-uint.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT randNegativeBinomial(100, .75) FROM numbers(5)
+```
+
+Result:
+
+``` text
+┌─randNegativeBinomial(100, 0.75)─┐
+│                              33 │
+│                              32 │
+│                              39 │
+│                              40 │
+│                              50 │
+└─────────────────────────────────┘
+```
+
+
+
+## randPoisson
+
+Return random number based on [Poisson distribution](https://en.wikipedia.org/wiki/Poisson_distribution).
+
+**Syntax**
+
+``` sql
+randPoisson(n)
+```
+
+**Arguments**
+
+- `n` - `UInt64` mean number of occurrences.
+
+**Returned value**
+
+- Pseudo-random number.
+
+Type: [UInt64](/docs/en/sql-reference/data-types/int-uint.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT randPoisson(10) FROM numbers(5)
+```
+
+Result:
+
+``` text
+┌─randPoisson(10)─┐
+│               8 │
+│               8 │
+│               7 │
+│              10 │
+│               6 │
+└─────────────────┘
+```
+
+
+
+## randBernoulli
+
+Return random number based on [Bernoulli distribution](https://en.wikipedia.org/wiki/Bernoulli_distribution).
+
+**Syntax**
+
+``` sql
+randBernoulli(probability)
+```
+
+**Arguments**
+
+- `probability` - `Float64` - probability of success (values in `0...1` range only).
+
+**Returned value**
+
+- Pseudo-random number.
+
+Type: [UInt64](/docs/en/sql-reference/data-types/int-uint.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT randBernoulli(.75) FROM numbers(5)
+```
+
+Result:
+
+``` text
+┌─randBernoulli(0.75)─┐
+│                   1 │
+│                   1 │
+│                   0 │
+│                   1 │
+│                   1 │
+└─────────────────────┘
+```
+
+
+
+## randExponential
+
+Return random number based on [exponential distribution](https://en.wikipedia.org/wiki/Exponential_distribution).
+
+**Syntax**
+
+``` sql
+randExponential(lambda)
+```
+
+**Arguments**
+
+- `lambda` - `Float64` lambda value.
+
+**Returned value**
+
+- Pseudo-random number.
+
+Type: [Float64](/docs/en/sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT randExponential(1/10) FROM numbers(5)
+```
+
+Result:
+
+``` text
+┌─randExponential(divide(1, 10))─┐
+│              44.71628934340778 │
+│              4.211013337903262 │
+│             10.809402553207766 │
+│              15.63959406553284 │
+│             1.8148392319860158 │
+└────────────────────────────────┘
+```
+
+
+
+## randChiSquared
+
+Return random number based on [Chi-square distribution](https://en.wikipedia.org/wiki/Chi-squared_distribution) - a distribution of a sum of the squares of k independent standard normal random variables.
+
+**Syntax**
+
+``` sql
+randChiSquared(degree_of_freedom)
+```
+
+**Arguments**
+
+- `degree_of_freedom` - `Float64` degree of freedom.
+
+**Returned value**
+
+- Pseudo-random number.
+
+Type: [Float64](/docs/en/sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT randChiSquared(10) FROM numbers(5)
+```
+
+Result:
+
+``` text
+┌─randChiSquared(10)─┐
+│ 10.015463656521543 │
+│  9.621799919882768 │
+│   2.71785015634699 │
+│ 11.128188665931908 │
+│  4.902063104425469 │
+└────────────────────┘
+```
+
+
+
+## randStudentT
+
+Return random number based on [Student's t-distribution](https://en.wikipedia.org/wiki/Student%27s_t-distribution).
+
+**Syntax**
+
+``` sql
+randStudentT(degree_of_freedom)
+```
+
+**Arguments**
+
+- `degree_of_freedom` - `Float64` degree of freedom.
+
+**Returned value**
+
+- Pseudo-random number.
+
+Type: [Float64](/docs/en/sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT randStudentT(10) FROM numbers(5)
+```
+
+Result:
+
+``` text
+┌─────randStudentT(10)─┐
+│   1.2217309938538725 │
+│   1.7941971681200541 │
+│ -0.28192176076784664 │
+│   0.2508897721303792 │
+│  -2.7858432909761186 │
+└──────────────────────┘
+```
+
+
+
+## randFisherF
+
+Return random number based on [F-distribution](https://en.wikipedia.org/wiki/F-distribution).
+
+**Syntax**
+
+``` sql
+randFisherF(d1, d2)
+```
+
+**Arguments**
+
+- `d1` - `Float64` d1 degree of freedom in `X = (S1 / d1) / (S2 / d2)`,
+- `d2` - `Float64` d2 degree of freedom in `X = (S1 / d1) / (S2 / d2)`,
+
+**Returned value**
+
+- Pseudo-random number.
+
+Type: [Float64](/docs/en/sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT randFisherF(10, 3) FROM numbers(5)
+```
+
+Result:
+
+``` text
+┌──randFisherF(10, 3)─┐
+│   7.286287504216609 │
+│ 0.26590779413050386 │
+│ 0.22207610901168987 │
+│  0.7953362728449572 │
+│ 0.19278885985221572 │
+└─────────────────────┘
+```
+
+
+
+
 # Random Functions for Working with Strings
 
 ## randomString

docs/en/sql-reference/functions/type-conversion-functions.md

@@ -14,7 +14,7 @@ ClickHouse has the [same behavior as C++ programs](https://en.cppreference.com/w
 
 ## toInt(8\|16\|32\|64\|128\|256)
 
-Converts an input value to the [Int](../../sql-reference/data-types/int-uint.md) data type. This function family includes:
+Converts an input value to the [Int](/docs/en/sql-reference/data-types/int-uint.md) data type. This function family includes:
 
 -   `toInt8(expr)` — Results in the `Int8` data type.
 -   `toInt16(expr)` — Results in the `Int16` data type.
@@ -25,7 +25,7 @@ Converts an input value to the [Int](../../sql-reference/data-types/int-uint.md)
 
 **Arguments**
 
--   `expr` — [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a number or a string with the decimal representation of a number. Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+-   `expr` — [Expression](/docs/en/sql-reference/syntax.md/#syntax-expressions) returning a number or a string with the decimal representation of a number. Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
 
 **Returned value**
 
@@ -33,7 +33,7 @@ Integer value in the `Int8`, `Int16`, `Int32`, `Int64`, `Int128` or `Int256` dat
 
 Functions use [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning they truncate fractional digits of numbers.
 
-The behavior of functions for the [NaN and Inf](../../sql-reference/data-types/float.md#data_type-float-nan-inf) arguments is undefined. Remember about [numeric conversions issues](#numeric-conversion-issues), when using the functions.
+The behavior of functions for the [NaN and Inf](/docs/en/sql-reference/data-types/float.md/#data_type-float-nan-inf) arguments is undefined. Remember about [numeric conversions issues](#numeric-conversion-issues), when using the functions.
 
 **Example**
 
@@ -114,7 +114,7 @@ Result:
 
 ## toUInt(8\|16\|32\|64\|256)
 
-Converts an input value to the [UInt](../../sql-reference/data-types/int-uint.md) data type. This function family includes:
+Converts an input value to the [UInt](/docs/en/sql-reference/data-types/int-uint.md) data type. This function family includes:
 
 -   `toUInt8(expr)` — Results in the `UInt8` data type.
 -   `toUInt16(expr)` — Results in the `UInt16` data type.
@@ -124,7 +124,7 @@ Converts an input value to the [UInt](../../sql-reference/data-types/int-uint.md
 
 **Arguments**
 
--   `expr` — [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a number or a string with the decimal representation of a number. Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+-   `expr` — [Expression](/docs/en/sql-reference/syntax.md/#syntax-expressions) returning a number or a string with the decimal representation of a number. Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
 
 **Returned value**
 
@@ -132,7 +132,7 @@ Integer value in the `UInt8`, `UInt16`, `UInt32`, `UInt64` or `UInt256` data typ
 
 Functions use [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning they truncate fractional digits of numbers.
 
-The behavior of functions for negative arguments and for the [NaN and Inf](../../sql-reference/data-types/float.md#data_type-float-nan-inf) arguments is undefined. If you pass a string with a negative number, for example `'-32'`, ClickHouse raises an exception. Remember about [numeric conversions issues](#numeric-conversion-issues), when using the functions.
+The behavior of functions for negative arguments and for the [NaN and Inf](/docs/en/sql-reference/data-types/float.md/#data_type-float-nan-inf) arguments is undefined. If you pass a string with a negative number, for example `'-32'`, ClickHouse raises an exception. Remember about [numeric conversions issues](#numeric-conversion-issues), when using the functions.
 
 **Example**
 
@@ -166,7 +166,111 @@ Result:
 
 ## toDate
 
-Alias: `DATE`.
+Converts the argument to `Date` data type. 
+
+If the argument is `DateTime` or `DateTime64`, it truncates it, leaving the date component of the DateTime:
+```sql
+SELECT
+    now() AS x,
+    toDate(x)
+```
+```response
+┌───────────────────x─┬─toDate(now())─┐
+│ 2022-12-30 13:44:17 │    2022-12-30 │
+└─────────────────────┴───────────────┘
+```
+
+If the argument is a string, it is parsed as Date or DateTime. If it was parsed as DateTime, the date component is being used:
+```sql
+SELECT
+    toDate('2022-12-30') AS x,
+    toTypeName(x)
+```
+```response
+┌──────────x─┬─toTypeName(toDate('2022-12-30'))─┐
+│ 2022-12-30 │ Date                             │
+└────────────┴──────────────────────────────────┘
+
+1 row in set. Elapsed: 0.001 sec. 
+```
+```sql
+SELECT
+    toDate('2022-12-30 01:02:03') AS x,
+    toTypeName(x)
+```
+```response
+┌──────────x─┬─toTypeName(toDate('2022-12-30 01:02:03'))─┐
+│ 2022-12-30 │ Date                                      │
+└────────────┴───────────────────────────────────────────┘
+```
+
+If the argument is a number and it looks like a UNIX timestamp (is greater than 65535), it is interpreted as a DateTime, then truncated to Date in the current timezone. The timezone argument can be specified as a second argument of the function. The truncation to Date depends on the timezone:
+
+```sql
+SELECT
+    now() AS current_time,
+    toUnixTimestamp(current_time) AS ts,
+    toDateTime(ts) AS time_Amsterdam,
+    toDateTime(ts, 'Pacific/Apia') AS time_Samoa,
+    toDate(time_Amsterdam) AS date_Amsterdam,
+    toDate(time_Samoa) AS date_Samoa,
+    toDate(ts) AS date_Amsterdam_2,
+    toDate(ts, 'Pacific/Apia') AS date_Samoa_2
+```
+```response
+Row 1:
+──────
+current_time:     2022-12-30 13:51:54
+ts:               1672404714
+time_Amsterdam:   2022-12-30 13:51:54
+time_Samoa:       2022-12-31 01:51:54
+date_Amsterdam:   2022-12-30
+date_Samoa:       2022-12-31
+date_Amsterdam_2: 2022-12-30
+date_Samoa_2:     2022-12-31
+```
+
+The example above demonstrates how the same UNIX timestamp can be interpreted as different dates in different time zones.
+
+If the argument is a number and it is smaller than 65536, it is interpreted as the number of days since 1970-01-01 (a UNIX day) and converted to Date. It corresponds to the internal numeric representation of the `Date` data type. Example:
+
+```sql
+SELECT toDate(12345)
+```
+```response
+┌─toDate(12345)─┐
+│    2003-10-20 │
+└───────────────┘
+```
+
+This conversion does not depend on timezones.
+
+If the argument does not fit in the range of the Date type, it results in an implementation-defined behavior, that can saturate to the maximum supported date or overflow:
+```sql
+SELECT toDate(10000000000.)
+```
+```response
+┌─toDate(10000000000.)─┐
+│           2106-02-07 │
+└──────────────────────┘
+```
+
+The function `toDate` can be also written in alternative forms:
+
+```sql
+SELECT
+    now() AS time,
+    toDate(time),
+    DATE(time),
+    CAST(time, 'Date')
+```
+```response
+┌────────────────time─┬─toDate(now())─┬─DATE(now())─┬─CAST(now(), 'Date')─┐
+│ 2022-12-30 13:54:58 │    2022-12-30 │  2022-12-30 │          2022-12-30 │
+└─────────────────────┴───────────────┴─────────────┴─────────────────────┘
+```
+
+Have a nice day working with dates and times.
 
 ## toDateOrZero
 
@@ -184,7 +288,7 @@ Alias: `DATE`.
 
 ## toDate32
 
-Converts the argument to the [Date32](../../sql-reference/data-types/date32.md) data type. If the value is outside the range returns the border values supported by `Date32`. If the argument has [Date](../../sql-reference/data-types/date.md) type, borders of `Date` are taken into account.
+Converts the argument to the [Date32](/docs/en/sql-reference/data-types/date32.md) data type. If the value is outside the range, `toDate32` returns the border values supported by `Date32`. If the argument has [Date](/docs/en/sql-reference/data-types/date.md) type, borders of `Date` are taken into account.
 
 **Syntax**
 
@@ -194,13 +298,13 @@ toDate32(expr)
 
 **Arguments**
 
--   `expr` — The value. [String](../../sql-reference/data-types/string.md), [UInt32](../../sql-reference/data-types/int-uint.md) or [Date](../../sql-reference/data-types/date.md).
+-   `expr` — The value. [String](/docs/en/sql-reference/data-types/string.md), [UInt32](/docs/en/sql-reference/data-types/int-uint.md) or [Date](/docs/en/sql-reference/data-types/date.md).
 
 **Returned value**
 
 -   A calendar date.
 
-Type: [Date32](../../sql-reference/data-types/date32.md).
+Type: [Date32](/docs/en/sql-reference/data-types/date32.md).
 
 **Example**
 
@@ -242,7 +346,7 @@ SELECT toDate32(toDate('1899-01-01')) AS value, toTypeName(value);
 
 ## toDate32OrZero
 
-The same as [toDate32](#todate32) but returns the min value of [Date32](../../sql-reference/data-types/date32.md) if invalid argument is received.
+The same as [toDate32](#todate32) but returns the min value of [Date32](/docs/en/sql-reference/data-types/date32.md) if an invalid argument is received.
 
 **Example**
 
@@ -262,7 +366,7 @@ Result:
 
 ## toDate32OrNull
 
-The same as [toDate32](#todate32) but returns `NULL` if invalid argument is received.
+The same as [toDate32](#todate32) but returns `NULL` if an invalid argument is received.
 
 **Example**
 
@@ -282,7 +386,7 @@ Result:
 
 ## toDate32OrDefault
 
-Converts the argument to the [Date32](../../sql-reference/data-types/date32.md) data type. If the value is outside the range returns the lower border value supported by `Date32`. If the argument has [Date](../../sql-reference/data-types/date.md) type, borders of `Date` are taken into account. Returns default value if invalid argument is received.
+Converts the argument to the [Date32](/docs/en/sql-reference/data-types/date32.md) data type. If the value is outside the range, `toDate32OrDefault` returns the lower border value supported by `Date32`. If the argument has [Date](/docs/en/sql-reference/data-types/date.md) type, borders of `Date` are taken into account. Returns default value if an invalid argument is received.
 
 **Example**
 
@@ -304,7 +408,7 @@ Result:
 
 ## toDateTime64
 
-Converts the argument to the [DateTime64](../../sql-reference/data-types/datetime64.md) data type.
+Converts the argument to the [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) data type.
 
 **Syntax**
 
@@ -314,7 +418,7 @@ toDateTime64(expr, scale, [timezone])
 
 **Arguments**
 
--   `expr` — The value. [String](../../sql-reference/data-types/string.md), [UInt32](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [DateTime](../../sql-reference/data-types/datetime.md).
+-   `expr` — The value. [String](/docs/en/sql-reference/data-types/string.md), [UInt32](/docs/en/sql-reference/data-types/int-uint.md), [Float](/docs/en/sql-reference/data-types/float.md) or [DateTime](/docs/en/sql-reference/data-types/datetime.md).
 -   `scale` - Tick size (precision): 10<sup>-precision</sup> seconds. Valid range: [ 0 : 9 ].
 -   `timezone` - Time zone of the specified datetime64 object.
 
@@ -322,7 +426,7 @@ toDateTime64(expr, scale, [timezone])
 
 -   A calendar date and time of day, with sub-second precision.
 
-Type: [DateTime64](../../sql-reference/data-types/datetime64.md).
+Type: [DateTime64](/docs/en/sql-reference/data-types/datetime64.md).
 
 **Example**
 
@@ -378,7 +482,7 @@ SELECT toDateTime64('2019-01-01 00:00:00', 3, 'Asia/Istanbul') AS value, toTypeN
 
 ## toDecimal(32\|64\|128\|256)
 
-Converts `value` to the [Decimal](../../sql-reference/data-types/decimal.md) data type with precision of `S`. The `value` can be a number or a string. The `S` (scale) parameter specifies the number of decimal places.
+Converts `value` to the [Decimal](/docs/en/sql-reference/data-types/decimal.md) data type with precision of `S`. The `value` can be a number or a string. The `S` (scale) parameter specifies the number of decimal places.
 
 -   `toDecimal32(value, S)`
 -   `toDecimal64(value, S)`
@@ -387,7 +491,7 @@ Converts `value` to the [Decimal](../../sql-reference/data-types/decimal.md) dat
 
 ## toDecimal(32\|64\|128\|256)OrNull
 
-Converts an input string to a [Nullable(Decimal(P,S))](../../sql-reference/data-types/decimal.md) data type value. This family of functions include:
+Converts an input string to a [Nullable(Decimal(P,S))](/docs/en/sql-reference/data-types/decimal.md) data type value. This family of functions includes:
 
 -   `toDecimal32OrNull(expr, S)` — Results in `Nullable(Decimal32(S))` data type.
 -   `toDecimal64OrNull(expr, S)` — Results in `Nullable(Decimal64(S))` data type.
@@ -398,7 +502,7 @@ These functions should be used instead of `toDecimal*()` functions, if you prefe
 
 **Arguments**
 
--   `expr` — [Expression](../../sql-reference/syntax.md#syntax-expressions), returns a value in the [String](../../sql-reference/data-types/string.md) data type. ClickHouse expects the textual representation of the decimal number. For example, `'1.111'`.
+-   `expr` — [Expression](/docs/en/sql-reference/syntax.md/#syntax-expressions), returns a value in the [String](/docs/en/sql-reference/data-types/string.md) data type. ClickHouse expects the textual representation of the decimal number. For example, `'1.111'`.
 -   `S` — Scale, the number of decimal places in the resulting value.
 
 **Returned value**
@@ -441,7 +545,7 @@ Result:
 
 ## toDecimal(32\|64\|128\|256)OrDefault
 
-Converts an input string to a [Decimal(P,S)](../../sql-reference/data-types/decimal.md) data type value. This family of functions include:
+Converts an input string to a [Decimal(P,S)](/docs/en/sql-reference/data-types/decimal.md) data type value. This family of functions includes:
 
 -   `toDecimal32OrDefault(expr, S)` — Results in `Decimal32(S)` data type.
 -   `toDecimal64OrDefault(expr, S)` — Results in `Decimal64(S)` data type.
@@ -452,7 +556,7 @@ These functions should be used instead of `toDecimal*()` functions, if you prefe
 
 **Arguments**
 
--   `expr` — [Expression](../../sql-reference/syntax.md#syntax-expressions), returns a value in the [String](../../sql-reference/data-types/string.md) data type. ClickHouse expects the textual representation of the decimal number. For example, `'1.111'`.
+-   `expr` — [Expression](/docs/en/sql-reference/syntax.md/#syntax-expressions), returns a value in the [String](/docs/en/sql-reference/data-types/string.md) data type. ClickHouse expects the textual representation of the decimal number. For example, `'1.111'`.
 -   `S` — Scale, the number of decimal places in the resulting value.
 
 **Returned value**
@@ -494,7 +598,7 @@ Result:
 
 ## toDecimal(32\|64\|128\|256)OrZero
 
-Converts an input value to the [Decimal(P,S)](../../sql-reference/data-types/decimal.md) data type. This family of functions include:
+Converts an input value to the [Decimal(P,S)](/docs/en/sql-reference/data-types/decimal.md) data type. This family of functions includes:
 
 -   `toDecimal32OrZero( expr, S)` — Results in `Decimal32(S)` data type.
 -   `toDecimal64OrZero( expr, S)` — Results in `Decimal64(S)` data type.
@@ -505,7 +609,7 @@ These functions should be used instead of `toDecimal*()` functions, if you prefe
 
 **Arguments**
 
--   `expr` — [Expression](../../sql-reference/syntax.md#syntax-expressions), returns a value in the [String](../../sql-reference/data-types/string.md) data type. ClickHouse expects the textual representation of the decimal number. For example, `'1.111'`.
+-   `expr` — [Expression](/docs/en/sql-reference/syntax.md/#syntax-expressions), returns a value in the [String](/docs/en/sql-reference/data-types/string.md) data type. ClickHouse expects the textual representation of the decimal number. For example, `'1.111'`.
 -   `S` — Scale, the number of decimal places in the resulting value.
 
 **Returned value**
@@ -564,7 +668,7 @@ YYYY-MM-DD hh:mm:ss
 
 As an exception, if converting from UInt32, Int32, UInt64, or Int64 numeric types to Date, and if the number is greater than or equal to 65536, the number is interpreted as a Unix timestamp (and not as the number of days) and is rounded to the date. This allows support for the common occurrence of writing ‘toDate(unix_timestamp)’, which otherwise would be an error and would require writing the more cumbersome ‘toDate(toDateTime(unix_timestamp))’.
 
-Conversion between a date and date with time is performed the natural way: by adding a null time or dropping the time.
+Conversion between a date and a date with time is performed the natural way: by adding a null time or dropping the time.
 
 Conversion between numeric types uses the same rules as assignments between different numeric types in C++.
 
@@ -643,15 +747,15 @@ These functions accept a string and interpret the bytes placed at the beginning
 
 ## reinterpretAsString
 
-This function accepts a number or date or date with time, and returns a string containing bytes representing the corresponding value in host order (little endian). Null bytes are dropped from the end. For example, a UInt32 type value of 255 is a string that is one byte long.
+This function accepts a number or date or date with time and returns a string containing bytes representing the corresponding value in host order (little endian). Null bytes are dropped from the end. For example, a UInt32 type value of 255 is a string that is one byte long.
 
 ## reinterpretAsFixedString
 
-This function accepts a number or date or date with time, and returns a FixedString containing bytes representing the corresponding value in host order (little endian). Null bytes are dropped from the end. For example, a UInt32 type value of 255 is a FixedString that is one byte long.
+This function accepts a number or date or date with time and returns a FixedString containing bytes representing the corresponding value in host order (little endian). Null bytes are dropped from the end. For example, a UInt32 type value of 255 is a FixedString that is one byte long.
 
 ## reinterpretAsUUID
 
-Accepts 16 bytes string and returns UUID containing bytes representing the corresponding value in network byte order (big-endian). If the string isn't long enough, the function works as if the string is padded with the necessary number of null bytes to the end. If the string longer than 16 bytes, the extra bytes at the end are ignored.
+Accepts 16 bytes string and returns UUID containing bytes representing the corresponding value in network byte order (big-endian). If the string isn't long enough, the function works as if the string is padded with the necessary number of null bytes to the end. If the string is longer than 16 bytes, the extra bytes at the end are ignored.
 
 **Syntax**
 
@@ -661,11 +765,11 @@ reinterpretAsUUID(fixed_string)
 
 **Arguments**
 
--   `fixed_string` — Big-endian byte string. [FixedString](../../sql-reference/data-types/fixedstring.md#fixedstring).
+-   `fixed_string` — Big-endian byte string. [FixedString](/docs/en/sql-reference/data-types/fixedstring.md/#fixedstring).
 
 **Returned value**
 
--   The UUID type value. [UUID](../../sql-reference/data-types/uuid.md#uuid-data-type).
+-   The UUID type value. [UUID](/docs/en/sql-reference/data-types/uuid.md/#uuid-data-type).
 
 **Examples**
 
@@ -718,7 +822,7 @@ reinterpret(x, type)
 **Arguments**
 
 -   `x` — Any type.
--   `type` — Destination type. [String](../../sql-reference/data-types/string.md).
+-   `type` — Destination type. [String](/docs/en/sql-reference/data-types/string.md).
 
 **Returned value**
 
@@ -757,7 +861,7 @@ x::t
 **Arguments**
 
 -   `x` — A value to convert. May be of any type.
--   `T` — The name of the target data type. [String](../../sql-reference/data-types/string.md).
+-   `T` — The name of the target data type. [String](/docs/en/sql-reference/data-types/string.md).
 -   `t` — The target data type.
 
 **Returned value**
@@ -806,9 +910,9 @@ Result:
 └─────────────────────┴─────────────────────┴────────────┴─────────────────────┴───────────────────────────┘
 ```
 
-Conversion to FixedString(N) only works for arguments of type [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
+Conversion to FixedString(N) only works for arguments of type [String](/docs/en/sql-reference/data-types/string.md) or [FixedString](/docs/en/sql-reference/data-types/fixedstring.md).
 
-Type conversion to [Nullable](../../sql-reference/data-types/nullable.md) and back is supported.
+Type conversion to [Nullable](/docs/en/sql-reference/data-types/nullable.md) and back is supported.
 
 **Example**
 
@@ -844,7 +948,7 @@ Result:
 
 **See also**
 
--   [cast_keep_nullable](../../operations/settings/settings.md#cast_keep_nullable) setting
+-   [cast_keep_nullable](/docs/en/operations/settings/settings.md/#cast_keep_nullable) setting
 
 ## accurateCast(x, T)
 
@@ -882,7 +986,7 @@ Code: 70. DB::Exception: Received from localhost:9000. DB::Exception: Value in c
 
 ## accurateCastOrNull(x, T)
 
-Converts input value `x` to the specified data type `T`. Always returns [Nullable](../../sql-reference/data-types/nullable.md) type and returns [NULL](../../sql-reference/syntax.md#null-literal) if the casted value is not representable in the target type.
+Converts input value `x` to the specified data type `T`. Always returns [Nullable](/docs/en/sql-reference/data-types/nullable.md) type and returns [NULL](/docs/en/sql-reference/syntax.md/#null-literal) if the casted value is not representable in the target type.
 
 **Syntax**
 
@@ -991,7 +1095,7 @@ Result:
 
 ## toInterval(Year\|Quarter\|Month\|Week\|Day\|Hour\|Minute\|Second)
 
-Converts a Number type argument to an [Interval](../../sql-reference/data-types/special-data-types/interval.md) data type.
+Converts a Number type argument to an [Interval](/docs/en/sql-reference/data-types/special-data-types/interval.md) data type.
 
 **Syntax**
 
@@ -1039,7 +1143,7 @@ Result:
 ## parseDateTimeBestEffort
 ## parseDateTime32BestEffort
 
-Converts a date and time in the [String](../../sql-reference/data-types/string.md) representation to [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) data type.
+Converts a date and time in the [String](/docs/en/sql-reference/data-types/string.md) representation to [DateTime](/docs/en/sql-reference/data-types/datetime.md/#data_type-datetime) data type.
 
 The function parses [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601), [RFC 1123 - 5.2.14 RFC-822 Date and Time Specification](https://tools.ietf.org/html/rfc1123#page-55), ClickHouse’s and some other date and time formats.
 
@@ -1051,8 +1155,8 @@ parseDateTimeBestEffort(time_string [, time_zone])
 
 **Arguments**
 
--   `time_string` — String containing a date and time to convert. [String](../../sql-reference/data-types/string.md).
+-   `time_string` — String containing a date and time to convert. [String](/docs/en/sql-reference/data-types/string.md).
--   `time_zone` — Time zone. The function parses `time_string` according to the time zone. [String](../../sql-reference/data-types/string.md).
+-   `time_zone` — Time zone. The function parses `time_string` according to the time zone. [String](/docs/en/sql-reference/data-types/string.md).
 
 **Supported non-standard formats**
 
@@ -1175,7 +1279,7 @@ Same as [parseDateTimeBestEffortUS](#parsedatetimebesteffortUS) function except
 
 ## parseDateTime64BestEffort
 
-Same as [parseDateTimeBestEffort](#parsedatetimebesteffort) function but also parse milliseconds and microseconds and returns [DateTime](../../sql-reference/functions/type-conversion-functions.md#data_type-datetime) data type.
+Same as [parseDateTimeBestEffort](#parsedatetimebesteffort) function but also parse milliseconds and microseconds and returns [DateTime](/docs/en/sql-reference/functions/type-conversion-functions.md/#data_type-datetime) data type.
 
 **Syntax**
 
@@ -1185,13 +1289,13 @@ parseDateTime64BestEffort(time_string [, precision [, time_zone]])
 
 **Parameters**
 
--   `time_string` — String containing a date or date with time to convert. [String](../../sql-reference/data-types/string.md).
+-   `time_string` — String containing a date or date with time to convert. [String](/docs/en/sql-reference/data-types/string.md).
--   `precision` — Required precision. `3` — for milliseconds, `6` — for microseconds. Default — `3`. Optional. [UInt8](../../sql-reference/data-types/int-uint.md).
+-   `precision` — Required precision. `3` — for milliseconds, `6` — for microseconds. Default — `3`. Optional. [UInt8](/docs/en/sql-reference/data-types/int-uint.md).
--   `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../../sql-reference/data-types/string.md).
+-   `time_zone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](/docs/en/sql-reference/data-types/string.md).
 
 **Returned value**
 
--   `time_string` converted to the [DateTime](../../sql-reference/data-types/datetime.md) data type.
+-   `time_string` converted to the [DateTime](/docs/en/sql-reference/data-types/datetime.md) data type.
 
 **Examples**
 
@@ -1242,7 +1346,7 @@ Same as for [parseDateTime64BestEffort](#parsedatetime64besteffort), except that
 
 ## toLowCardinality
 
-Converts input parameter to the [LowCardinality](../../sql-reference/data-types/lowcardinality.md) version of same data type.
+Converts input parameter to the [LowCardinality](/docs/en/sql-reference/data-types/lowcardinality.md) version of same data type.
 
 To convert data from the `LowCardinality` data type use the [CAST](#type_conversion_function-cast) function. For example, `CAST(x as String)`.
 
@@ -1254,7 +1358,7 @@ toLowCardinality(expr)
 
 **Arguments**
 
--   `expr` — [Expression](../../sql-reference/syntax.md#syntax-expressions) resulting in one of the [supported data types](../../sql-reference/data-types/index.md#data_types).
+-   `expr` — [Expression](/docs/en/sql-reference/syntax.md/#syntax-expressions) resulting in one of the [supported data types](/docs/en/sql-reference/data-types/index.md/#data_types).
 
 **Returned values**
 
@@ -1388,7 +1492,7 @@ formatRow(format, x, y, ...)
 
 **Arguments**
 
--   `format` — Text format. For example, [CSV](../../interfaces/formats.md#csv), [TSV](../../interfaces/formats.md#tabseparated).
+-   `format` — Text format. For example, [CSV](/docs/en/interfaces/formats.md/#csv), [TSV](/docs/en/interfaces/formats.md/#tabseparated).
 -   `x`,`y`, ... — Expressions.
 
 **Returned value**
@@ -1429,7 +1533,7 @@ formatRowNoNewline(format, x, y, ...)
 
 **Arguments**
 
--   `format` — Text format. For example, [CSV](../../interfaces/formats.md#csv), [TSV](../../interfaces/formats.md#tabseparated).
+-   `format` — Text format. For example, [CSV](/docs/en/interfaces/formats.md/#csv), [TSV](/docs/en/interfaces/formats.md/#tabseparated).
 -   `x`,`y`, ... — Expressions.
 
 **Returned value**
@@ -1457,7 +1561,7 @@ Result:
 
 ## snowflakeToDateTime
 
-Extracts time from [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as [DateTime](../data-types/datetime.md) format.
+Extracts time from [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as [DateTime](/docs/en/sql-reference/data-types/datetime.md) format.
 
 **Syntax**
 
@@ -1467,12 +1571,12 @@ snowflakeToDateTime(value [, time_zone])
 
 **Parameters**
 
--   `value` — Snowflake ID. [Int64](../data-types/int-uint.md).
+-   `value` — Snowflake ID. [Int64](/docs/en/sql-reference/data-types/int-uint.md).
--   `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../../sql-reference/data-types/string.md).
+-   `time_zone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](/docs/en/sql-reference/data-types/string.md).
 
 **Returned value**
 
--  Input value converted to the [DateTime](../data-types/datetime.md) data type.
+-  Input value converted to the [DateTime](/docs/en/sql-reference/data-types/datetime.md) data type.
 
 **Example**
 
@@ -1493,7 +1597,7 @@ Result:
 
 ## snowflakeToDateTime64
 
-Extracts time from [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as [DateTime64](../data-types/datetime64.md) format.
+Extracts time from [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) format.
 
 **Syntax**
 
@@ -1503,12 +1607,12 @@ snowflakeToDateTime64(value [, time_zone])
 
 **Parameters**
 
--   `value` — Snowflake ID. [Int64](../data-types/int-uint.md).
+-   `value` — Snowflake ID. [Int64](/docs/en/sql-reference/data-types/int-uint.md).
--   `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../../sql-reference/data-types/string.md).
+-   `time_zone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](/docs/en/sql-reference/data-types/string.md).
 
 **Returned value**
 
--  Input value converted to the [DateTime64](../data-types/datetime64.md) data type.
+-  Input value converted to the [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) data type.
 
 **Example**
 
@@ -1529,7 +1633,7 @@ Result:
 
 ## dateTimeToSnowflake
 
-Converts [DateTime](../data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
+Converts [DateTime](/docs/en/sql-reference/data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
 
 **Syntax**
 
@@ -1539,11 +1643,11 @@ dateTimeToSnowflake(value)
 
 **Parameters**
 
--   `value` — Date and time. [DateTime](../../sql-reference/data-types/datetime.md).
+-   `value` — Date and time. [DateTime](/docs/en/sql-reference/data-types/datetime.md).
 
 **Returned value**
 
--   Input value converted to the [Int64](../data-types/int-uint.md) data type as the first Snowflake ID at that time.
+-   Input value converted to the [Int64](/docs/en/sql-reference/data-types/int-uint.md) data type as the first Snowflake ID at that time.
 
 **Example**
 
@@ -1563,7 +1667,7 @@ Result:
 
 ## dateTime64ToSnowflake
 
-Convert [DateTime64](../data-types/datetime64.md) to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
+Convert [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
 
 **Syntax**
 
@@ -1573,11 +1677,11 @@ dateTime64ToSnowflake(value)
 
 **Parameters**
 
--   `value` — Date and time. [DateTime64](../../sql-reference/data-types/datetime64.md).
+-   `value` — Date and time. [DateTime64](/docs/en/sql-reference/data-types/datetime64.md).
 
 **Returned value**
 
--   Input value converted to the [Int64](../data-types/int-uint.md) data type as the first Snowflake ID at that time.
+-   Input value converted to the [Int64](/docs/en/sql-reference/data-types/int-uint.md) data type as the first Snowflake ID at that time.
 
 **Example**
 

docs/en/sql-reference/statements/select/from.md

@@ -21,12 +21,11 @@ Subquery is another `SELECT` query that may be specified in parenthesis inside `
 
 When `FINAL` is specified, ClickHouse fully merges the data before returning the result and thus performs all data transformations that happen during merges for the given table engine.
 
-It is applicable when selecting data from tables that use the [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md)-engine family. Also supported for:
+It is applicable when selecting data from ReplacingMergeTree, SummingMergeTree, AggregatingMergeTree, CollapsingMergeTree and VersionedCollapsingMergeTree tables.
 
--   [Replicated](../../../engines/table-engines/mergetree-family/replication.md) versions of `MergeTree` engines.
+`SELECT` queries with `FINAL` are executed in parallel. The [max_final_threads](../../../operations/settings/settings.md#max-final-threads) setting limits the number of threads used.
--   [View](../../../engines/table-engines/special/view.md), [Buffer](../../../engines/table-engines/special/buffer.md), [Distributed](../../../engines/table-engines/special/distributed.md), and [MaterializedView](../../../engines/table-engines/special/materializedview.md) engines that operate over other engines, provided they were created over `MergeTree`-engine tables.
 
-Now `SELECT` queries with `FINAL` are executed in parallel and slightly faster. But there are drawbacks (see below). The [max_final_threads](../../../operations/settings/settings.md#max-final-threads) setting limits the number of threads used.
+There are drawbacks to using `FINAL` (see below). 
 
 ### Drawbacks
 

docs/zh/engines/table-engines/log-family/index.md

@@ -11,7 +11,7 @@ sidebar_position: 29
 这系列的引擎有：
 
 -   [StripeLog](stripelog.md)
--   [日志](log.md)
+-   [Log](log.md)
 -   [TinyLog](tinylog.md)
 
 ## 共同属性 {#table_engines-log-engine-family-common-properties}

programs/benchmark/Benchmark.cpp

@@ -683,7 +683,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
             ("confidence", value<size_t>()->default_value(5), "set the level of confidence for T-test [0=80%, 1=90%, 2=95%, 3=98%, 4=99%, 5=99.5%(default)")
             ("query_id", value<std::string>()->default_value(""), "")
             ("max-consecutive-errors", value<size_t>()->default_value(0), "set number of allowed consecutive errors")
-            ("continue_on_errors", "continue testing even if a query fails")
+            ("ignore-error,continue_on_errors", "continue testing even if a query fails")
             ("reconnect", "establish new connection for every query")
             ("client-side-time", "display the time including network communication instead of server-side time; note that for server versions before 22.8 we always display client-side time")
         ;
@@ -738,7 +738,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
             options["query_id"].as<std::string>(),
             options["query"].as<std::string>(),
             options["max-consecutive-errors"].as<size_t>(),
-            options.count("continue_on_errors"),
+            options.count("ignore-error"),
             options.count("reconnect"),
             options.count("client-side-time"),
             print_stacktrace,

programs/client/CMakeLists.txt

@@ -13,6 +13,10 @@ set (CLICKHOUSE_CLIENT_LINK
         string_utils
 )
 
+if (TARGET ch_rust::skim)
+    list(APPEND CLICKHOUSE_CLIENT_LINK PRIVATE ch_rust::skim)
+endif()
+
 # Always use internal readpassphrase
 list(APPEND CLICKHOUSE_CLIENT_LINK PRIVATE readpassphrase)
 

programs/client/Client.cpp

@@ -30,9 +30,10 @@
 
 #include <IO/ReadBufferFromString.h>
 #include <IO/ReadHelpers.h>
-#include <IO/WriteHelpers.h>
-#include <IO/WriteBufferFromOStream.h>
 #include <IO/UseSSL.h>
+#include <IO/WriteBufferFromOStream.h>
+#include <IO/WriteHelpers.h>
+#include <IO/copyData.h>
 
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTDropQuery.h>
@@ -41,6 +42,8 @@
 #include <Parsers/ASTInsertQuery.h>
 #include <Parsers/ASTSelectQuery.h>
 
+#include <Processors/Transforms/getSourceFromASTInsertQuery.h>
+
 #include <Interpreters/InterpreterSetQuery.h>
 
 #include <Functions/registerFunctions.h>
@@ -827,6 +830,20 @@ bool Client::processWithFuzzing(const String & full_query)
         WriteBufferFromOStream ast_buf(std::cout, 4096);
         formatAST(*query, ast_buf, false /*highlight*/);
         ast_buf.next();
+        if (const auto * insert = query->as<ASTInsertQuery>())
+        {
+            /// For inserts with data it's really useful to have the data itself available in the logs, as formatAST doesn't print it
+            if (insert->hasInlinedData())
+            {
+                String bytes;
+                {
+                    auto read_buf = getReadBufferFromASTInsertQuery(query);
+                    WriteBufferFromString write_buf(bytes);
+                    copyData(*read_buf, write_buf);
+                }
+                std::cout << std::endl << bytes;
+            }
+        }
         std::cout << std::endl << std::endl;
 
         try

programs/install/Install.cpp

@@ -1051,18 +1051,12 @@ namespace
         return pid;
     }
 
-    int stop(const fs::path & pid_file, bool force, bool do_not_kill, unsigned max_tries)
+    bool sendSignalAndWaitForStop(const fs::path & pid_file, int signal, unsigned max_tries, unsigned wait_ms, const char * signal_name)
     {
-        if (force && do_not_kill)
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Specified flags are incompatible");
-
         int pid = isRunning(pid_file);
 
         if (!pid)
-            return 0;
+            return true;
-
-        int signal = force ? SIGKILL : SIGTERM;
-        const char * signal_name = force ? "kill" : "terminate";
 
         if (0 == kill(pid, signal))
             fmt::print("Sent {} signal to process with pid {}.\n", signal_name, pid);
@@ -1078,46 +1072,51 @@ namespace
                 fmt::print("Server stopped\n");
                 break;
             }
-            sleepForSeconds(1);
+            sleepForMilliseconds(wait_ms);
         }
 
-        if (try_num == max_tries)
+        return try_num < max_tries;
+    }
+
+    int stop(const fs::path & pid_file, bool force, bool do_not_kill, unsigned max_tries)
+    {
+        if (force && do_not_kill)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Specified flags are incompatible");
+
+        int signal = force ? SIGKILL : SIGTERM;
+        const char * signal_name = force ? "kill" : "terminate";
+
+        if (sendSignalAndWaitForStop(pid_file, signal, max_tries, 1000, signal_name))
+            return 0;
+
+        int pid = isRunning(pid_file);
+        if (!pid)
+            return 0;
+
+        if (do_not_kill)
         {
-            if (do_not_kill)
+            fmt::print("Process (pid = {}) is still running. Will not try to kill it.\n", pid);
-            {
+            return 1;
-                fmt::print("Process (pid = {}) is still running. Will not try to kill it.\n", pid);
-                return 1;
-            }
-
-            fmt::print("Will terminate forcefully (pid = {}).\n", pid);
-            if (0 == kill(pid, 9))
-                fmt::print("Sent kill signal (pid = {}).\n", pid);
-            else
-                throwFromErrno("Cannot send kill signal", ErrorCodes::SYSTEM_ERROR);
-
-            /// Wait for the process (100 seconds).
-            constexpr size_t num_kill_check_tries = 1000;
-            constexpr size_t kill_check_delay_ms = 100;
-            for (size_t i = 0; i < num_kill_check_tries; ++i)
-            {
-                fmt::print("Waiting for server to be killed\n");
-                if (!isRunning(pid_file))
-                {
-                    fmt::print("Server exited\n");
-                    break;
-                }
-                sleepForMilliseconds(kill_check_delay_ms);
-            }
-
-            if (isRunning(pid_file))
-            {
-                throw Exception(ErrorCodes::CANNOT_KILL,
-                    "The server process still exists after {} tries (delay: {} ms)",
-                    num_kill_check_tries, kill_check_delay_ms);
-            }
         }
 
-        return 0;
+        /// Send termination signal again, the server will receive it and immediately terminate.
+        fmt::print("Will send the termination signal again to force the termination (pid = {}).\n", pid);
+        if (sendSignalAndWaitForStop(pid_file, signal, std::min(10U, max_tries), 1000, signal_name))
+            return 0;
+
+        /// Send kill signal. Total wait is 100 seconds.
+        constexpr size_t num_kill_check_tries = 1000;
+        constexpr size_t kill_check_delay_ms = 100;
+        fmt::print("Will terminate forcefully (pid = {}).\n", pid);
+        if (sendSignalAndWaitForStop(pid_file, SIGKILL, num_kill_check_tries, kill_check_delay_ms, signal_name))
+            return 0;
+
+        if (!isRunning(pid_file))
+            return 0;
+
+        throw Exception(ErrorCodes::CANNOT_KILL,
+            "The server process still exists after {} tries (delay: {} ms)",
+            num_kill_check_tries, kill_check_delay_ms);
     }
 }
 

programs/local/CMakeLists.txt

@@ -18,6 +18,10 @@ if(NOT CLICKHOUSE_ONE_SHARED)
     target_link_libraries(clickhouse-local-lib PRIVATE clickhouse-server-lib)
 endif()
 
+if (TARGET ch_rust::skim)
+    target_link_libraries(clickhouse-local-lib PRIVATE ch_rust::skim)
+endif()
+
 # Always use internal readpassphrase
 target_link_libraries(clickhouse-local-lib PRIVATE readpassphrase)
 

programs/local/LocalServer.cpp

@@ -207,7 +207,7 @@ void LocalServer::tryInitPath()
 
     global_context->setPath(path);
 
-    global_context->setTemporaryStorage(path + "tmp", "", 0);
+    global_context->setTemporaryStoragePath(path + "tmp/", 0);
     global_context->setFlagsPath(path + "flags");
 
     global_context->setUserFilesPath(""); // user's files are everywhere

programs/server/Server.cpp

@@ -70,6 +70,8 @@
 #include <QueryPipeline/ConnectionCollector.h>
 #include <Dictionaries/registerDictionaries.h>
 #include <Disks/registerDisks.h>
+#include <IO/Resource/registerSchedulerNodes.h>
+#include <IO/Resource/registerResourceManagers.h>
 #include <Common/Config/ConfigReloader.h>
 #include <Server/HTTPHandlerFactory.h>
 #include "MetricsTransmitter.h"
@@ -203,46 +205,6 @@ int mainEntryClickHouseServer(int argc, char ** argv)
 namespace
 {
 
-void setupTmpPath(Poco::Logger * log, const std::string & path)
-try
-{
-    LOG_DEBUG(log, "Setting up {} to store temporary data in it", path);
-
-    fs::create_directories(path);
-
-    /// Clearing old temporary files.
-    fs::directory_iterator dir_end;
-    size_t unknown_files = 0;
-    for (fs::directory_iterator it(path); it != dir_end; ++it)
-    {
-        if (it->is_regular_file() && startsWith(it->path().filename(), "tmp"))
-        {
-            LOG_DEBUG(log, "Removing old temporary file {}", it->path().string());
-            fs::remove(it->path());
-        }
-        else
-        {
-            unknown_files++;
-            if (unknown_files < 100)
-                LOG_DEBUG(log, "Found unknown {} {} in temporary path",
-                    it->is_regular_file() ? "file" : (it->is_directory() ? "directory" : "element"),
-                    it->path().string());
-        }
-    }
-
-    if (unknown_files)
-        LOG_DEBUG(log, "Found {} unknown files in temporary path", unknown_files);
-}
-catch (...)
-{
-    DB::tryLogCurrentException(
-        log,
-        fmt::format(
-            "Caught exception while setup temporary path: {}. It is ok to skip this exception as cleaning old temporary files is not "
-            "necessary",
-            path));
-}
-
 size_t waitServersToFinish(std::vector<DB::ProtocolServerAdapter> & servers, size_t seconds_to_wait)
 {
     const size_t sleep_max_ms = 1000 * seconds_to_wait;
@@ -715,6 +677,8 @@ try
     registerDisks(/* global_skip_access_check= */ false);
     registerFormats();
     registerRemoteFileMetadatas();
+    registerSchedulerNodes();
+    registerResourceManagers();
 
     CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::getVersionRevision());
     CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
@@ -739,6 +703,13 @@ try
     global_context->addWarningMessage("Server was built with sanitizer. It will work slowly.");
 #endif
 
+    const auto memory_amount = getMemoryAmount();
+
+    LOG_INFO(log, "Available RAM: {}; physical cores: {}; logical cores: {}.",
+        formatReadableSizeWithBinarySuffix(memory_amount),
+        getNumberOfPhysicalCPUCores(),  // on ARM processors it can show only enabled at current moment cores
+        std::thread::hardware_concurrency());
+
     sanityChecks(*this);
 
     // Initialize global thread pool. Do it before we fetch configs from zookeeper
@@ -812,8 +783,6 @@ try
 
     Settings::checkNoSettingNamesAtTopLevel(config(), config_path);
 
-    const auto memory_amount = getMemoryAmount();
-
 #if defined(OS_LINUX)
     std::string executable_path = getExecutablePath();
 
@@ -1009,13 +978,21 @@ try
     LOG_TRACE(log, "Initialized DateLUT with time zone '{}'.", DateLUT::instance().getTimeZone());
 
     /// Storage with temporary data for processing of heavy queries.
+    if (auto temporary_policy = config().getString("tmp_policy", ""); !temporary_policy.empty())
+    {
+        size_t max_size = config().getUInt64("max_temporary_data_on_disk_size", 0);
+        global_context->setTemporaryStoragePolicy(temporary_policy, max_size);
+    }
+    else if (auto temporary_cache = config().getString("temporary_data_in_cache", ""); !temporary_cache.empty())
+    {
+        size_t max_size = config().getUInt64("max_temporary_data_on_disk_size", 0);
+        global_context->setTemporaryStorageInCache(temporary_cache, max_size);
+    }
+    else
     {
         std::string temporary_path = config().getString("tmp_path", path / "tmp/");
-        std::string temporary_policy = config().getString("tmp_policy", "");
         size_t max_size = config().getUInt64("max_temporary_data_on_disk_size", 0);
-        const VolumePtr & volume = global_context->setTemporaryStorage(temporary_path, temporary_policy, max_size);
+        global_context->setTemporaryStoragePath(temporary_path, max_size);
-        for (const DiskPtr & disk : volume->getDisks())
-            setupTmpPath(log, disk->getPath());
     }
 
     /** Directory with 'flags': files indicating temporary settings for the server set by system administrator.
@@ -1072,8 +1049,8 @@ try
         bool continue_if_corrupted = config().getBool("merge_tree_metadata_cache.continue_if_corrupted", false);
         try
         {
-            LOG_DEBUG(
+            LOG_DEBUG(log, "Initializing MergeTree metadata cache, lru_cache_size: {} continue_if_corrupted: {}",
-                log, "Initializing merge tree metadata cache lru_cache_size:{} continue_if_corrupted:{}", size, continue_if_corrupted);
+                ReadableSize(size), continue_if_corrupted);
             global_context->initializeMergeTreeMetadataCache(path_str + "/" + "rocksdb", size);
         }
         catch (...)
@@ -1289,6 +1266,11 @@ try
                 global_context->getDistributedSchedulePool().increaseThreadsCount(new_pool_size);
             }
 
+            if (config->has("resources"))
+            {
+                global_context->getResourceManager()->updateConfiguration(*config);
+            }
+
             if (!initial_loading)
             {
                 /// We do not load ZooKeeper configuration on the first config loading
@@ -1417,7 +1399,7 @@ try
     }
     catch (...)
     {
-        tryLogCurrentException(log);
+        tryLogCurrentException(log, "Caught exception while setting up access control.");
         throw;
     }
 
@@ -1750,13 +1732,6 @@ try
         main_config_reloader->start();
         access_control.startPeriodicReloading();
 
-        {
-            LOG_INFO(log, "Available RAM: {}; physical cores: {}; logical cores: {}.",
-                formatReadableSizeWithBinarySuffix(memory_amount),
-                getNumberOfPhysicalCPUCores(),  // on ARM processors it can show only enabled at current moment cores
-                std::thread::hardware_concurrency());
-        }
-
         /// try to load dictionaries immediately, throw on error and die
         try
         {

programs/server/config.d/graphite.xml

@@ -0,0 +1 @@
+../../../tests/config/config.d/graphite.xml

rust/CMakeLists.txt

@@ -39,5 +39,21 @@ function(clickhouse_import_crate)
     corrosion_import_crate(NO_STD ${ARGN})
 endfunction()
 
-add_subdirectory (BLAKE3)
+# Add crate from the build directory.
-add_subdirectory (skim)
+#
+# Our crates has configuration files:
+# - config for cargo (see config.toml.in)
+# - and possibly config for build (build.rs.in)
+#
+# And to avoid overlaps different builds for one source directory, crate will
+# be copied from source directory to the binary directory.
+file(COPY ".cargo" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
+function(add_rust_subdirectory src)
+    set(dst "${CMAKE_CURRENT_BINARY_DIR}/${src}")
+    message(STATUS "Copy ${src} to ${dst}")
+    file(COPY "${src}" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
+    add_subdirectory("${dst}" "${dst}")
+endfunction()
+
+add_rust_subdirectory (BLAKE3)
+add_rust_subdirectory (skim)

rust/skim/build.rs.in

@@ -5,4 +5,5 @@ fn main() {
     }
     build.compile("skim");
     println!("cargo:rerun-if-changed=src/lib.rs");
+    println!("cargo:rerun-if-changed=.cargo/config.toml");
 }

rust/skim/include/skim.h

@@ -87,4 +87,4 @@ private:
 } // namespace cxxbridge1
 } // namespace rust
 
-::rust::String skim(::std::vector<::std::string> const &words);
+::rust::String skim(::std::string const &prefix, ::std::vector<::std::string> const &words);

rust/skim/src/lib.rs

@@ -5,7 +5,7 @@ use cxx::{CxxString, CxxVector};
 #[cxx::bridge]
 mod ffi {
     extern "Rust" {
-        fn skim(words: &CxxVector<CxxString>) -> Result<String>;
+        fn skim(prefix: &CxxString, words: &CxxVector<CxxString>) -> Result<String>;
     }
 }
 
@@ -18,7 +18,7 @@ impl SkimItem for Item {
     }
 }
 
-fn skim(words: &CxxVector<CxxString>) -> Result<String, String> {
+fn skim(prefix: &CxxString, words: &CxxVector<CxxString>) -> Result<String, String> {
     // Let's check is terminal available. To avoid panic.
     if let Err(err) = TermInfo::from_env() {
         return Err(format!("{}", err));
@@ -26,6 +26,7 @@ fn skim(words: &CxxVector<CxxString>) -> Result<String, String> {
 
     let options = SkimOptionsBuilder::default()
         .height(Some("30%"))
+        .query(Some(prefix.to_str().unwrap()))
         .tac(true)
         .tiebreak(Some("-score".to_string()))
         .build()

src/Access/SettingsProfileElement.cpp

@@ -75,6 +75,10 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A
     }
 }
 
+bool SettingsProfileElement::isConstraint() const
+{
+    return this->writability || !this->min_value.isNull() || !this->max_value.isNull();
+}
 
 std::shared_ptr<ASTSettingsProfileElement> SettingsProfileElement::toAST() const
 {
@@ -213,7 +217,7 @@ SettingsConstraints SettingsProfileElements::toSettingsConstraints(const AccessC
 {
     SettingsConstraints res{access_control};
     for (const auto & elem : *this)
-        if (!elem.setting_name.empty() && elem.setting_name != ALLOW_BACKUP_SETTING_NAME)
+        if (!elem.setting_name.empty() && elem.isConstraint() && elem.setting_name != ALLOW_BACKUP_SETTING_NAME)
             res.set(
                 elem.setting_name,
                 elem.min_value,

src/Access/SettingsProfileElement.h

@@ -44,6 +44,8 @@ struct SettingsProfileElement
     std::shared_ptr<ASTSettingsProfileElement> toAST() const;
     std::shared_ptr<ASTSettingsProfileElement> toASTWithNames(const AccessControl & access_control) const;
 
+    bool isConstraint() const;
+
 private:
     void init(const ASTSettingsProfileElement & ast, const AccessControl * access_control);
 };

src/AggregateFunctions/AggregateFunctionAggThrow.cpp

@@ -49,14 +49,16 @@ private:
 
 public:
     AggregateFunctionThrow(const DataTypes & argument_types_, const Array & parameters_, Float64 throw_probability_)
-        : IAggregateFunctionDataHelper(argument_types_, parameters_), throw_probability(throw_probability_) {}
+        : IAggregateFunctionDataHelper(argument_types_, parameters_, createResultType())
+        , throw_probability(throw_probability_)
+    {}
 
     String getName() const override
     {
         return "aggThrow";
     }
 
-    DataTypePtr getReturnType() const override
+    static DataTypePtr createResultType()
     {
         return std::make_shared<DataTypeUInt8>();
     }

src/AggregateFunctions/AggregateFunctionAnalysisOfVariance.h

@@ -37,10 +37,10 @@ class AggregateFunctionAnalysisOfVariance final : public IAggregateFunctionDataH
 {
 public:
     explicit AggregateFunctionAnalysisOfVariance(const DataTypes & arguments, const Array & params)
-    : IAggregateFunctionDataHelper(arguments, params)
+        : IAggregateFunctionDataHelper(arguments, params, createResultType())
     {}
 
-    DataTypePtr getReturnType() const override
+    DataTypePtr createResultType() const
     {
         DataTypes types {std::make_shared<DataTypeNumber<Float64>>(), std::make_shared<DataTypeNumber<Float64>>() };
         Strings names {"f_statistic", "p_value"};

src/AggregateFunctions/AggregateFunctionArgMinMax.h

@@ -38,7 +38,6 @@ template <typename Data>
 class AggregateFunctionArgMinMax final : public IAggregateFunctionDataHelper<Data, AggregateFunctionArgMinMax<Data>>
 {
 private:
-    const DataTypePtr & type_res;
     const DataTypePtr & type_val;
     const SerializationPtr serialization_res;
     const SerializationPtr serialization_val;
@@ -47,10 +46,9 @@ private:
 
 public:
     AggregateFunctionArgMinMax(const DataTypePtr & type_res_, const DataTypePtr & type_val_)
-        : Base({type_res_, type_val_}, {})
+        : Base({type_res_, type_val_}, {}, type_res_)
-        , type_res(this->argument_types[0])
         , type_val(this->argument_types[1])
-        , serialization_res(type_res->getDefaultSerialization())
+        , serialization_res(type_res_->getDefaultSerialization())
         , serialization_val(type_val->getDefaultSerialization())
     {
         if (!type_val->isComparable())
@@ -63,11 +61,6 @@ public:
         return StringRef(Data::ValueData_t::name()) == StringRef("min") ? "argMin" : "argMax";
     }
 
-    DataTypePtr getReturnType() const override
-    {
-        return type_res;
-    }
-
     void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override
     {
         if (this->data(place).value.changeIfBetter(*columns[1], row_num, arena))

src/AggregateFunctions/AggregateFunctionArray.h

@@ -30,7 +30,7 @@ private:
 
 public:
     AggregateFunctionArray(AggregateFunctionPtr nested_, const DataTypes & arguments, const Array & params_)
-        : IAggregateFunctionHelper<AggregateFunctionArray>(arguments, params_)
+        : IAggregateFunctionHelper<AggregateFunctionArray>(arguments, params_, createResultType(nested_))
         , nested_func(nested_), num_arguments(arguments.size())
     {
         assert(parameters == nested_func->getParameters());
@@ -44,9 +44,9 @@ public:
         return nested_func->getName() + "Array";
     }
 
-    DataTypePtr getReturnType() const override
+    static DataTypePtr createResultType(const AggregateFunctionPtr & nested_)
     {
-        return nested_func->getReturnType();
+        return nested_->getResultType();
     }
 
     const IAggregateFunction & getBaseAggregateFunctionWithSameStateRepresentation() const override

src/AggregateFunctions/AggregateFunctionAvg.h

@@ -10,6 +10,7 @@
 #include <AggregateFunctions/IAggregateFunction.h>
 #include <AggregateFunctions/AggregateFunctionSum.h>
 #include <Core/DecimalFunctions.h>
+#include <Core/IResolvedFunction.h>
 
 #include "config.h"
 
@@ -83,10 +84,20 @@ public:
     using Fraction = AvgFraction<Numerator, Denominator>;
 
     explicit AggregateFunctionAvgBase(const DataTypes & argument_types_,
-        UInt32 num_scale_ = 0, UInt32 denom_scale_ = 0)
+                                      UInt32 num_scale_ = 0, UInt32 denom_scale_ = 0)
-        : Base(argument_types_, {}), num_scale(num_scale_), denom_scale(denom_scale_) {}
+        : Base(argument_types_, {}, createResultType())
+        , num_scale(num_scale_)
+        , denom_scale(denom_scale_)
+    {}
 
-    DataTypePtr getReturnType() const override { return std::make_shared<DataTypeNumber<Float64>>(); }
+    AggregateFunctionAvgBase(const DataTypes & argument_types_, const DataTypePtr & result_type_,
+                             UInt32 num_scale_ = 0, UInt32 denom_scale_ = 0)
+        : Base(argument_types_, {}, result_type_)
+        , num_scale(num_scale_)
+        , denom_scale(denom_scale_)
+    {}
+
+    DataTypePtr createResultType() const { return std::make_shared<DataTypeNumber<Float64>>(); }
 
     bool allocatesMemoryInArena() const override { return false; }
 
@@ -135,7 +146,7 @@ public:
         for (const auto & argument : this->argument_types)
             can_be_compiled &= canBeNativeType(*argument);
 
-        auto return_type = getReturnType();
+        auto return_type = this->getResultType();
         can_be_compiled &= canBeNativeType(*return_type);
 
         return can_be_compiled;

src/AggregateFunctions/AggregateFunctionBitwise.h

@@ -97,11 +97,12 @@ class AggregateFunctionBitwise final : public IAggregateFunctionDataHelper<Data,
 {
 public:
     explicit AggregateFunctionBitwise(const DataTypePtr & type)
-        : IAggregateFunctionDataHelper<Data, AggregateFunctionBitwise<T, Data>>({type}, {}) {}
+        : IAggregateFunctionDataHelper<Data, AggregateFunctionBitwise<T, Data>>({type}, {}, createResultType())
+    {}
 
     String getName() const override { return Data::name(); }
 
-    DataTypePtr getReturnType() const override
+    static DataTypePtr createResultType()
     {
         return std::make_shared<DataTypeNumber<T>>();
     }
@@ -137,7 +138,7 @@ public:
 
     bool isCompilable() const override
     {
-        auto return_type = getReturnType();
+        auto return_type = this->getResultType();
         return canBeNativeType(*return_type);
     }
 
@@ -151,7 +152,7 @@ public:
     {
         llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
 
-        auto * return_type = toNativeType(b, getReturnType());
+        auto * return_type = toNativeType(b, this->getResultType());
 
         auto * value_ptr = aggregate_data_ptr;
         auto * value = b.CreateLoad(return_type, value_ptr);
@@ -166,7 +167,7 @@ public:
     {
         llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
 
-        auto * return_type = toNativeType(b, getReturnType());
+        auto * return_type = toNativeType(b, this->getResultType());
 
         auto * value_dst_ptr = aggregate_data_dst_ptr;
         auto * value_dst = b.CreateLoad(return_type, value_dst_ptr);
@@ -183,7 +184,7 @@ public:
     {
         llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
 
-        auto * return_type = toNativeType(b, getReturnType());
+        auto * return_type = toNativeType(b, this->getResultType());
         auto * value_ptr = aggregate_data_ptr;
 
         return b.CreateLoad(return_type, value_ptr);

src/AggregateFunctions/AggregateFunctionBoundingRatio.h

@@ -112,7 +112,7 @@ public:
     }
 
     explicit AggregateFunctionBoundingRatio(const DataTypes & arguments)
-        : IAggregateFunctionDataHelper<AggregateFunctionBoundingRatioData, AggregateFunctionBoundingRatio>(arguments, {})
+        : IAggregateFunctionDataHelper<AggregateFunctionBoundingRatioData, AggregateFunctionBoundingRatio>(arguments, {}, std::make_shared<DataTypeFloat64>())
     {
         const auto * x_arg = arguments.at(0).get();
         const auto * y_arg = arguments.at(1).get();
@@ -122,11 +122,6 @@ public:
                 ErrorCodes::BAD_ARGUMENTS);
     }
 
-    DataTypePtr getReturnType() const override
-    {
-        return std::make_shared<DataTypeFloat64>();
-    }
-
     bool allocatesMemoryInArena() const override { return false; }
 
     void add(AggregateDataPtr __restrict place, const IColumn ** columns, const size_t row_num, Arena *) const override

src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp

@@ -46,9 +46,9 @@ private:
     }
 
 public:
-    AggregateFunctionCategoricalIV(const DataTypes & arguments_, const Array & params_) :
+    AggregateFunctionCategoricalIV(const DataTypes & arguments_, const Array & params_)
-        IAggregateFunctionHelper<AggregateFunctionCategoricalIV>{arguments_, params_},
+        : IAggregateFunctionHelper<AggregateFunctionCategoricalIV>{arguments_, params_, createResultType()}
-        category_count{arguments_.size() - 1}
+        , category_count{arguments_.size() - 1}
     {
         // notice: argument types has been checked before
     }
@@ -121,7 +121,7 @@ public:
         buf.readStrict(place, sizeOfData());
     }
 
-    DataTypePtr getReturnType() const override
+    static DataTypePtr createResultType()
     {
         return std::make_shared<DataTypeArray>(
             std::make_shared<DataTypeNumber<Float64>>());

src/AggregateFunctions/AggregateFunctionCount.h

@@ -39,11 +39,13 @@ namespace ErrorCodes
 class AggregateFunctionCount final : public IAggregateFunctionDataHelper<AggregateFunctionCountData, AggregateFunctionCount>
 {
 public:
-    explicit AggregateFunctionCount(const DataTypes & argument_types_) : IAggregateFunctionDataHelper(argument_types_, {}) {}
+    explicit AggregateFunctionCount(const DataTypes & argument_types_)
+        : IAggregateFunctionDataHelper(argument_types_, {}, createResultType())
+    {}
 
     String getName() const override { return "count"; }
 
-    DataTypePtr getReturnType() const override
+    static DataTypePtr createResultType()
     {
         return std::make_shared<DataTypeUInt64>();
     }
@@ -167,7 +169,7 @@ public:
     {
         llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
 
-        auto * return_type = toNativeType(b, getReturnType());
+        auto * return_type = toNativeType(b, this->getResultType());
 
         auto * count_value_ptr = aggregate_data_ptr;
         auto * count_value = b.CreateLoad(return_type, count_value_ptr);
@@ -180,7 +182,7 @@ public:
     {
         llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
 
-        auto * return_type = toNativeType(b, getReturnType());
+        auto * return_type = toNativeType(b, this->getResultType());
 
         auto * count_value_dst_ptr = aggregate_data_dst_ptr;
         auto * count_value_dst = b.CreateLoad(return_type, count_value_dst_ptr);
@@ -197,7 +199,7 @@ public:
     {
         llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
 
-        auto * return_type = toNativeType(b, getReturnType());
+        auto * return_type = toNativeType(b, this->getResultType());
         auto * count_value_ptr = aggregate_data_ptr;
 
         return b.CreateLoad(return_type, count_value_ptr);
@@ -214,7 +216,7 @@ class AggregateFunctionCountNotNullUnary final
 {
 public:
     AggregateFunctionCountNotNullUnary(const DataTypePtr & argument, const Array & params)
-        : IAggregateFunctionDataHelper<AggregateFunctionCountData, AggregateFunctionCountNotNullUnary>({argument}, params)
+        : IAggregateFunctionDataHelper<AggregateFunctionCountData, AggregateFunctionCountNotNullUnary>({argument}, params, createResultType())
     {
         if (!argument->isNullable())
             throw Exception("Logical error: not Nullable data type passed to AggregateFunctionCountNotNullUnary", ErrorCodes::LOGICAL_ERROR);
@@ -222,7 +224,7 @@ public:
 
     String getName() const override { return "count"; }
 
-    DataTypePtr getReturnType() const override
+    static DataTypePtr createResultType()
     {
         return std::make_shared<DataTypeUInt64>();
     }
@@ -311,7 +313,7 @@ public:
     {
         llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
 
-        auto * return_type = toNativeType(b, getReturnType());
+        auto * return_type = toNativeType(b, this->getResultType());
 
         auto * is_null_value = b.CreateExtractValue(values[0], {1});
         auto * increment_value = b.CreateSelect(is_null_value, llvm::ConstantInt::get(return_type, 0), llvm::ConstantInt::get(return_type, 1));
@@ -327,7 +329,7 @@ public:
     {
         llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
 
-        auto * return_type = toNativeType(b, getReturnType());
+        auto * return_type = toNativeType(b, this->getResultType());
 
         auto * count_value_dst_ptr = aggregate_data_dst_ptr;
         auto * count_value_dst = b.CreateLoad(return_type, count_value_dst_ptr);
@@ -344,7 +346,7 @@ public:
     {
         llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
 
-        auto * return_type = toNativeType(b, getReturnType());
+        auto * return_type = toNativeType(b, this->getResultType());
         auto * count_value_ptr = aggregate_data_ptr;
 
         return b.CreateLoad(return_type, count_value_ptr);

src/AggregateFunctions/AggregateFunctionDeltaSum.h

@@ -31,7 +31,7 @@ class AggregationFunctionDeltaSum final
 {
 public:
     AggregationFunctionDeltaSum(const DataTypes & arguments, const Array & params)
-        : IAggregateFunctionDataHelper<AggregationFunctionDeltaSumData<T>, AggregationFunctionDeltaSum<T>>{arguments, params}
+        : IAggregateFunctionDataHelper<AggregationFunctionDeltaSumData<T>, AggregationFunctionDeltaSum<T>>{arguments, params, createResultType()}
     {}
 
     AggregationFunctionDeltaSum()
@@ -40,7 +40,7 @@ public:
 
     String getName() const override { return "deltaSum"; }
 
-    DataTypePtr getReturnType() const override { return std::make_shared<DataTypeNumber<T>>(); }
+    static DataTypePtr createResultType() { return std::make_shared<DataTypeNumber<T>>(); }
 
     bool allocatesMemoryInArena() const override { return false; }
 

src/AggregateFunctions/AggregateFunctionDeltaSumTimestamp.h

@@ -38,7 +38,7 @@ public:
         : IAggregateFunctionDataHelper<
             AggregationFunctionDeltaSumTimestampData<ValueType, TimestampType>,
             AggregationFunctionDeltaSumTimestamp<ValueType, TimestampType>
-        >{arguments, params}
+        >{arguments, params, createResultType()}
     {}
 
     AggregationFunctionDeltaSumTimestamp()
@@ -52,7 +52,7 @@ public:
 
     String getName() const override { return "deltaSumTimestamp"; }
 
-    DataTypePtr getReturnType() const override { return std::make_shared<DataTypeNumber<ValueType>>(); }
+    static DataTypePtr createResultType() { return std::make_shared<DataTypeNumber<ValueType>>(); }
 
     void NO_SANITIZE_UNDEFINED ALWAYS_INLINE add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override
     {

src/AggregateFunctions/AggregateFunctionDistinct.h

@@ -168,7 +168,7 @@ private:
 
 public:
     AggregateFunctionDistinct(AggregateFunctionPtr nested_func_, const DataTypes & arguments, const Array & params_)
-    : IAggregateFunctionDataHelper<Data, AggregateFunctionDistinct>(arguments, params_)
+    : IAggregateFunctionDataHelper<Data, AggregateFunctionDistinct>(arguments, params_, nested_func_->getResultType())
     , nested_func(nested_func_)
     , arguments_num(arguments.size())
     {
@@ -255,11 +255,6 @@ public:
         return nested_func->getName() + "Distinct";
     }
 
-    DataTypePtr getReturnType() const override
-    {
-        return nested_func->getReturnType();
-    }
-
     bool allocatesMemoryInArena() const override
     {
         return true;

src/AggregateFunctions/AggregateFunctionEntropy.h

@@ -92,14 +92,14 @@ private:
 
 public:
     explicit AggregateFunctionEntropy(const DataTypes & argument_types_)
-        : IAggregateFunctionDataHelper<EntropyData<Value>, AggregateFunctionEntropy<Value>>(argument_types_, {})
+        : IAggregateFunctionDataHelper<EntropyData<Value>, AggregateFunctionEntropy<Value>>(argument_types_, {}, createResultType())
         , num_args(argument_types_.size())
     {
     }
 
     String getName() const override { return "entropy"; }
 
-    DataTypePtr getReturnType() const override
+    static DataTypePtr createResultType()
     {
         return std::make_shared<DataTypeNumber<Float64>>();
     }

src/AggregateFunctions/AggregateFunctionExponentialMovingAverage.cpp

@@ -29,7 +29,7 @@ private:
 
 public:
     AggregateFunctionExponentialMovingAverage(const DataTypes & argument_types_, const Array & params)
-        : IAggregateFunctionDataHelper<ExponentiallySmoothedAverage, AggregateFunctionExponentialMovingAverage>(argument_types_, params)
+        : IAggregateFunctionDataHelper<ExponentiallySmoothedAverage, AggregateFunctionExponentialMovingAverage>(argument_types_, params, createResultType())
     {
         if (params.size() != 1)
             throw Exception{"Aggregate function " + getName() + " requires exactly one parameter: half decay time.",
@@ -43,7 +43,7 @@ public:
         return "exponentialMovingAverage";
     }
 
-    DataTypePtr getReturnType() const override
+    static DataTypePtr createResultType()
     {
         return std::make_shared<DataTypeNumber<Float64>>();
     }

src/AggregateFunctions/AggregateFunctionFactory.cpp

@@ -72,9 +72,12 @@ AggregateFunctionPtr AggregateFunctionFactory::get(
 {
     auto types_without_low_cardinality = convertLowCardinalityTypesToNested(argument_types);
 
-    /// If one of the types is Nullable, we apply aggregate function combinator "Null".
+    /// If one of the types is Nullable, we apply aggregate function combinator "Null" if it's not window function.
-
+    /// Window functions are not real aggregate functions. Applying combinators doesn't make sense for them,
-    if (std::any_of(types_without_low_cardinality.begin(), types_without_low_cardinality.end(),
+    /// they must handle the nullability themselves
+    auto properties = tryGetPropertiesImpl(name);
+    bool is_window_function = properties.has_value() && properties->is_window_function;
+    if (!is_window_function && std::any_of(types_without_low_cardinality.begin(), types_without_low_cardinality.end(),
         [](const auto & type) { return type->isNullable(); }))
     {
         AggregateFunctionCombinatorPtr combinator = AggregateFunctionCombinatorFactory::instance().tryFindSuffix("Null");

src/AggregateFunctions/AggregateFunctionFlameGraph.cpp

@@ -1,647 +0,0 @@
-#include <AggregateFunctions/AggregateFunctionFactory.h>
-#include <AggregateFunctions/IAggregateFunction.h>
-#include <AggregateFunctions/FactoryHelpers.h>
-#include <Common/HashTable/HashMap.h>
-#include <Common/SymbolIndex.h>
-#include <Common/ArenaAllocator.h>
-#include <Core/Settings.h>
-#include <Columns/ColumnArray.h>
-#include <Columns/ColumnString.h>
-#include <Columns/ColumnsNumber.h>
-#include <DataTypes/DataTypeArray.h>
-#include <DataTypes/DataTypeString.h>
-#include <DataTypes/DataTypesNumber.h>
-#include <IO/WriteHelpers.h>
-#include <IO/Operators.h>
-#include <filesystem>
-
-namespace DB
-{
-namespace ErrorCodes
-{
-    extern const int FUNCTION_NOT_ALLOWED;
-    extern const int NOT_IMPLEMENTED;
-    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
-}
-
-struct AggregateFunctionFlameGraphTree
-{
-    struct ListNode;
-
-    struct TreeNode
-    {
-        TreeNode * parent = nullptr;
-        ListNode * children = nullptr;
-        UInt64 ptr = 0;
-        size_t allocated = 0;
-    };
-
-    struct ListNode
-    {
-        ListNode * next = nullptr;
-        TreeNode * child = nullptr;
-    };
-
-    TreeNode root;
-
-    static ListNode * createChild(TreeNode * parent, UInt64 ptr, Arena * arena)
-    {
-
-        ListNode * list_node = reinterpret_cast<ListNode *>(arena->alloc(sizeof(ListNode)));
-        TreeNode * tree_node = reinterpret_cast<TreeNode *>(arena->alloc(sizeof(TreeNode)));
-
-        list_node->child = tree_node;
-        list_node->next = nullptr;
-
-        tree_node->parent =parent;
-        tree_node->children = nullptr;
-        tree_node->ptr = ptr;
-        tree_node->allocated = 0;
-
-        return list_node;
-    }
-
-    TreeNode * find(const UInt64 * stack, size_t stack_size, Arena * arena)
-    {
-        TreeNode * node = &root;
-        for (size_t i = 0; i < stack_size; ++i)
-        {
-            UInt64 ptr = stack[i];
-            if (ptr == 0)
-                break;
-
-            if (!node->children)
-            {
-                node->children = createChild(node, ptr, arena);
-                node = node->children->child;
-            }
-            else
-            {
-                ListNode * list = node->children;
-                while (list->child->ptr != ptr && list->next)
-                    list = list->next;
-
-                if (list->child->ptr != ptr)
-                {
-                    list->next = createChild(node, ptr, arena);
-                    list = list->next;
-                }
-
-                node = list->child;
-            }
-        }
-
-        return node;
-    }
-
-    static void append(DB::PaddedPODArray<UInt64> & values, DB::PaddedPODArray<UInt64> & offsets, std::vector<UInt64> & frame)
-    {
-        UInt64 prev = offsets.empty() ? 0 : offsets.back();
-        offsets.push_back(prev + frame.size());
-        for (UInt64 val : frame)
-            values.push_back(val);
-    }
-
-    struct Trace
-    {
-        using Frames = std::vector<UInt64>;
-
-        Frames frames;
-
-        /// The total number of bytes allocated for traces with the same prefix.
-        size_t allocated_total = 0;
-        /// This counter is relevant in case we want to filter some traces with small amount of bytes.
-        /// It shows the total number of bytes for *filtered* traces with the same prefix.
-        /// This is the value which is used in flamegraph.
-        size_t allocated_self = 0;
-    };
-
-    using Traces = std::vector<Trace>;
-
-    Traces dump(size_t max_depth, size_t min_bytes) const
-    {
-        Traces traces;
-        Trace::Frames frames;
-        std::vector<size_t> allocated_total;
-        std::vector<size_t> allocated_self;
-        std::vector<ListNode *> nodes;
-
-        nodes.push_back(root.children);
-        allocated_total.push_back(root.allocated);
-        allocated_self.push_back(root.allocated);
-
-        while (!nodes.empty())
-        {
-            if (nodes.back() == nullptr)
-            {
-                traces.push_back({frames, allocated_total.back(), allocated_self.back()});
-
-                nodes.pop_back();
-                allocated_total.pop_back();
-                allocated_self.pop_back();
-
-                /// We don't have root's frame so framers are empty in the end.
-                if (!frames.empty())
-                    frames.pop_back();
-
-                continue;
-            }
-
-            TreeNode * current = nodes.back()->child;
-            nodes.back() = nodes.back()->next;
-
-            bool enough_bytes = current->allocated >= min_bytes;
-            bool enough_depth = max_depth == 0 || nodes.size() < max_depth;
-
-            if (enough_bytes)
-            {
-                frames.push_back(current->ptr);
-                allocated_self.back() -= current->allocated;
-
-                if (enough_depth)
-                {
-                    allocated_total.push_back(current->allocated);
-                    allocated_self.push_back(current->allocated);
-                    nodes.push_back(current->children);
-                }
-                else
-                {
-                    traces.push_back({frames, current->allocated, current->allocated});
-                    frames.pop_back();
-                }
-            }
-        }
-
-        return traces;
-    }
-};
-
-static void insertData(DB::PaddedPODArray<UInt8> & chars, DB::PaddedPODArray<UInt64> & offsets, const char * pos, size_t length)
-{
-    const size_t old_size = chars.size();
-    const size_t new_size = old_size + length + 1;
-
-    chars.resize(new_size);
-    if (length)
-        memcpy(chars.data() + old_size, pos, length);
-    chars[old_size + length] = 0;
-    offsets.push_back(new_size);
-}
-
-/// Split str by line feed and write as separate row to ColumnString.
-static void fillColumn(DB::PaddedPODArray<UInt8> & chars, DB::PaddedPODArray<UInt64> & offsets, const std::string & str)
-{
-    size_t start = 0;
-    size_t end = 0;
-    size_t size = str.size();
-
-    while (end < size)
-    {
-        if (str[end] == '\n')
-        {
-            insertData(chars, offsets, str.data() + start, end - start);
-            start = end + 1;
-        }
-
-        ++end;
-    }
-
-    if (start < end)
-        insertData(chars, offsets, str.data() + start, end - start);
-}
-
-void dumpFlameGraph(
-    const AggregateFunctionFlameGraphTree::Traces & traces,
-    DB::PaddedPODArray<UInt8> & chars,
-    DB::PaddedPODArray<UInt64> & offsets)
-{
-    DB::WriteBufferFromOwnString out;
-
-    std::unordered_map<uintptr_t, size_t> mapping;
-
-#if defined(__ELF__) && !defined(OS_FREEBSD)
-    auto symbol_index_ptr = DB::SymbolIndex::instance();
-    const DB::SymbolIndex & symbol_index = *symbol_index_ptr;
-#endif
-
-    for (const auto & trace : traces)
-    {
-        if (trace.allocated_self == 0)
-            continue;
-
-        for (size_t i = 0; i < trace.frames.size(); ++i)
-        {
-            if (i)
-                out << ";";
-
-            const void * ptr = reinterpret_cast<const void *>(trace.frames[i]);
-
-#if defined(__ELF__) && !defined(OS_FREEBSD)
-            if (const auto * symbol = symbol_index.findSymbol(ptr))
-                writeString(demangle(symbol->name), out);
-            else
-                DB::writePointerHex(ptr, out);
-#else
-            DB::writePointerHex(ptr, out);
-#endif
-        }
-
-        out << ' ' << trace.allocated_self << "\n";
-    }
-
-    fillColumn(chars, offsets, out.str());
-}
-
-struct AggregateFunctionFlameGraphData
-{
-    struct Entry
-    {
-        AggregateFunctionFlameGraphTree::TreeNode * trace;
-        UInt64 size;
-        Entry * next = nullptr;
-    };
-
-    struct Pair
-    {
-        Entry * allocation = nullptr;
-        Entry * deallocation = nullptr;
-    };
-
-    using Entries = HashMap<UInt64, Pair>;
-
-    AggregateFunctionFlameGraphTree tree;
-    Entries entries;
-    Entry * free_list = nullptr;
-
-    Entry * alloc(Arena * arena)
-    {
-        if (free_list)
-        {
-            auto * res = free_list;
-            free_list = free_list->next;
-            return res;
-        }
-
-        return reinterpret_cast<Entry *>(arena->alloc(sizeof(Entry)));
-    }
-
-    void release(Entry * entry)
-    {
-        entry->next = free_list;
-        free_list = entry;
-    }
-
-    static void track(Entry * allocation)
-    {
-        auto * node = allocation->trace;
-        while (node)
-        {
-            node->allocated += allocation->size;
-            node = node->parent;
-        }
-    }
-
-    static void untrack(Entry * allocation)
-    {
-        auto * node = allocation->trace;
-        while (node)
-        {
-            node->allocated -= allocation->size;
-            node = node->parent;
-        }
-    }
-
-    static Entry * tryFindMatchAndRemove(Entry *& list, UInt64 size)
-    {
-        if (!list)
-            return nullptr;
-
-        if (list->size == size)
-        {
-            Entry * entry = list;
-            list = list->next;
-            return entry;
-        }
-        else
-        {
-            Entry * parent = list;
-            while (parent->next && parent->next->size != size)
-                parent = parent->next;
-
-            if (parent->next && parent->next->size == size)
-            {
-                Entry * entry = parent->next;
-                parent->next = entry->next;
-                return entry;
-            }
-
-            return nullptr;
-        }
-    }
-
-    void add(UInt64 ptr, Int64 size, const UInt64 * stack, size_t stack_size, Arena * arena)
-    {
-        /// In case if argument is nullptr, only track allocations.
-        if (ptr == 0)
-        {
-            if (size > 0)
-            {
-                auto * node = tree.find(stack, stack_size, arena);
-                Entry entry{.trace = node, .size = UInt64(size)};
-                track(&entry);
-            }
-
-            return;
-        }
-
-        auto & place = entries[ptr];
-        if (size > 0)
-        {
-            if (auto * deallocation = tryFindMatchAndRemove(place.deallocation, size))
-            {
-                release(deallocation);
-            }
-            else
-            {
-                auto * node = tree.find(stack, stack_size, arena);
-
-                auto * allocation = alloc(arena);
-                allocation->size = UInt64(size);
-                allocation->trace = node;
-
-                track(allocation);
-
-                allocation->next = place.allocation;
-                place.allocation = allocation;
-            }
-        }
-        else if (size < 0)
-        {
-            UInt64 abs_size = -size;
-            if (auto * allocation = tryFindMatchAndRemove(place.allocation, abs_size))
-            {
-                untrack(allocation);
-                release(allocation);
-            }
-            else
-            {
-                auto * deallocation = alloc(arena);
-                deallocation->size = abs_size;
-
-                deallocation->next = place.deallocation;
-                place.deallocation = deallocation;
-            }
-        }
-    }
-
-    void merge(const AggregateFunctionFlameGraphTree & other_tree, Arena * arena)
-    {
-        AggregateFunctionFlameGraphTree::Trace::Frames frames;
-        std::vector<AggregateFunctionFlameGraphTree::ListNode *> nodes;
-
-        nodes.push_back(other_tree.root.children);
-
-        while (!nodes.empty())
-        {
-            if (nodes.back() == nullptr)
-            {
-                nodes.pop_back();
-
-                /// We don't have root's frame so framers are empty in the end.
-                if (!frames.empty())
-                    frames.pop_back();
-
-                continue;
-            }
-
-            AggregateFunctionFlameGraphTree::TreeNode * current = nodes.back()->child;
-            nodes.back() = nodes.back()->next;
-
-            frames.push_back(current->ptr);
-
-            if (current->children)
-                nodes.push_back(current->children);
-            else
-            {
-                if (current->allocated)
-                    add(0, current->allocated, frames.data(), frames.size(), arena);
-
-                frames.pop_back();
-            }
-        }
-    }
-
-    void merge(const AggregateFunctionFlameGraphData & other, Arena * arena)
-    {
-        AggregateFunctionFlameGraphTree::Trace::Frames frames;
-        for (const auto & entry : other.entries)
-        {
-            for (auto * allocation = entry.value.second.allocation; allocation; allocation = allocation->next)
-            {
-                frames.clear();
-                const auto * node = allocation->trace;
-                while (node->ptr)
-                {
-                    frames.push_back(node->ptr);
-                    node = node->parent;
-                }
-
-                std::reverse(frames.begin(), frames.end());
-                add(entry.value.first, allocation->size, frames.data(), frames.size(), arena);
-                untrack(allocation);
-            }
-
-            for (auto * deallocation = entry.value.second.deallocation; deallocation; deallocation = deallocation->next)
-            {
-                add(entry.value.first, -Int64(deallocation->size), nullptr, 0, arena);
-            }
-        }
-
-        merge(other.tree, arena);
-    }
-
-    void dumpFlameGraph(
-        DB::PaddedPODArray<UInt8> & chars,
-        DB::PaddedPODArray<UInt64> & offsets,
-        size_t max_depth, size_t min_bytes) const
-    {
-        DB::dumpFlameGraph(tree.dump(max_depth, min_bytes), chars, offsets);
-    }
-};
-
-/// Aggregate function which builds a flamegraph using the list of stacktraces.
-/// The output is an array of strings which can be used by flamegraph.pl util.
-/// See https://github.com/brendangregg/FlameGraph
-///
-/// Syntax: flameGraph(traces, [size = 1], [ptr = 0])
-/// - trace : Array(UInt64), a stacktrace
-/// - size  : Int64, an allocation size (for memory profiling)
-/// - ptr   : UInt64, an allocation address
-/// In case if ptr != 0, a flameGraph will map allocations (size > 0) and deallocations (size < 0) with the same size and ptr.
-/// Only allocations which were not freed are shown. Not mapped deallocations are ignored.
-///
-/// Usage:
-///
-/// * Build a flamegraph based on CPU query profiler
-/// set query_profiler_cpu_time_period_ns=10000000;
-/// SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;
-/// clickhouse client --allow_introspection_functions=1
-///     -q "select arrayJoin(flameGraph(arrayReverse(trace))) from system.trace_log where trace_type = 'CPU' and query_id = 'xxx'"
-///     | ~/dev/FlameGraph/flamegraph.pl  > flame_cpu.svg
-///
-/// * Build a flamegraph based on memory query profiler, showing all allocations
-/// set memory_profiler_sample_probability=1, max_untracked_memory=1;
-/// SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;
-/// clickhouse client --allow_introspection_functions=1
-///     -q "select arrayJoin(flameGraph(trace, size)) from system.trace_log where trace_type = 'MemorySample' and query_id = 'xxx'"
-///     | ~/dev/FlameGraph/flamegraph.pl --countname=bytes --color=mem > flame_mem.svg
-///
-/// * Build a flamegraph based on memory query profiler, showing allocations which were not deallocated in query context
-/// set memory_profiler_sample_probability=1, max_untracked_memory=1, use_uncompressed_cache=1, merge_tree_max_rows_to_use_cache=100000000000, merge_tree_max_bytes_to_use_cache=1000000000000;
-/// SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;
-/// clickhouse client --allow_introspection_functions=1
-///     -q "select arrayJoin(flameGraph(trace, size, ptr)) from system.trace_log where trace_type = 'MemorySample' and query_id = 'xxx'"
-///     | ~/dev/FlameGraph/flamegraph.pl --countname=bytes --color=mem > flame_mem_untracked.svg
-///
-/// *  Build a flamegraph based on memory query profiler, showing active allocations at the fixed point of time
-/// set memory_profiler_sample_probability=1, max_untracked_memory=1;
-/// SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;
-/// 1. Memory usage per second
-/// select event_time, m, formatReadableSize(max(s) as m) from (select event_time, sum(size) over (order by event_time) as s from system.trace_log where query_id = 'xxx' and trace_type = 'MemorySample') group by event_time order by event_time;
-/// 2. Find a time point with maximal memory usage
-/// select argMax(event_time, s), max(s) from (select event_time, sum(size) over (order by event_time) as s from system.trace_log where query_id = 'xxx' and trace_type = 'MemorySample');
-/// 3. Fix active allocations at fixed point of time
-/// clickhouse client --allow_introspection_functions=1
-///      -q "select arrayJoin(flameGraph(trace, size, ptr)) from (select * from system.trace_log where trace_type = 'MemorySample' and query_id = 'xxx' and event_time <= 'yyy' order by event_time)"
-///      | ~/dev/FlameGraph/flamegraph.pl --countname=bytes --color=mem > flame_mem_time_point_pos.svg
-/// 4. Find deallocations at fixed point of time
-/// clickhouse client --allow_introspection_functions=1
-///      -q "select arrayJoin(flameGraph(trace, -size, ptr)) from (select * from system.trace_log where trace_type = 'MemorySample' and query_id = 'xxx' and event_time > 'yyy' order by event_time desc)"
-///      | ~/dev/FlameGraph/flamegraph.pl --countname=bytes --color=mem > flame_mem_time_point_neg.svg
-class AggregateFunctionFlameGraph final : public IAggregateFunctionDataHelper<AggregateFunctionFlameGraphData, AggregateFunctionFlameGraph>
-{
-public:
-    explicit AggregateFunctionFlameGraph(const DataTypes & argument_types_)
-        : IAggregateFunctionDataHelper<AggregateFunctionFlameGraphData, AggregateFunctionFlameGraph>(argument_types_, {})
-    {}
-
-    String getName() const override { return "flameGraph"; }
-
-    DataTypePtr getReturnType() const override
-    {
-        return std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
-    }
-
-    bool allocatesMemoryInArena() const override { return true; }
-
-    void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override
-    {
-        const auto * trace = typeid_cast<const ColumnArray *>(columns[0]);
-
-        const auto & trace_offsets = trace->getOffsets();
-        const auto & trace_values = typeid_cast<const ColumnUInt64 *>(&trace->getData())->getData();
-        UInt64 prev_offset = 0;
-        if (row_num)
-            prev_offset = trace_offsets[row_num - 1];
-        UInt64 trace_size = trace_offsets[row_num] - prev_offset;
-
-        Int64 allocated = 1;
-        if (argument_types.size() >= 2)
-        {
-            const auto & sizes = typeid_cast<const ColumnInt64 *>(columns[1])->getData();
-            allocated = sizes[row_num];
-        }
-
-        UInt64 ptr = 0;
-        if (argument_types.size() >= 3)
-        {
-            const auto & ptrs = typeid_cast<const ColumnUInt64 *>(columns[2])->getData();
-            ptr = ptrs[row_num];
-        }
-
-        this->data(place).add(ptr, allocated, trace_values.data() + prev_offset, trace_size, arena);
-    }
-
-    void addManyDefaults(
-        AggregateDataPtr __restrict /*place*/,
-        const IColumn ** /*columns*/,
-        size_t /*length*/,
-        Arena * /*arena*/) const override
-    {
-    }
-
-    void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override
-    {
-        this->data(place).merge(this->data(rhs), arena);
-    }
-
-    void serialize(ConstAggregateDataPtr __restrict, WriteBuffer &, std::optional<size_t> /* version */) const override
-    {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Serialization for function flameGraph is not implemented.");
-    }
-
-    void deserialize(AggregateDataPtr __restrict, ReadBuffer &, std::optional<size_t> /* version */, Arena *) const override
-    {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Deserialization for function flameGraph is not implemented.");
-    }
-
-    void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
-    {
-        auto & array = assert_cast<ColumnArray &>(to);
-        auto & str = assert_cast<ColumnString &>(array.getData());
-
-        this->data(place).dumpFlameGraph(str.getChars(), str.getOffsets(), 0, 0);
-
-        array.getOffsets().push_back(str.size());
-    }
-};
-
-static void check(const std::string & name, const DataTypes & argument_types, const Array & params)
-{
-    assertNoParameters(name, params);
-
-    if (argument_types.empty() || argument_types.size() > 3)
-        throw Exception(
-            ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
-            "Aggregate function {} requires 1 to 3 arguments : trace, [size = 1], [ptr = 0]",
-            name);
-
-    auto ptr_type = std::make_shared<DataTypeUInt64>();
-    auto trace_type = std::make_shared<DataTypeArray>(ptr_type);
-    auto size_type = std::make_shared<DataTypeInt64>();
-
-    if (!argument_types[0]->equals(*trace_type))
-        throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-            "First argument (trace) for function {} must be Array(UInt64), but it has type {}",
-            name, argument_types[0]->getName());
-
-    if (argument_types.size() >= 2 && !argument_types[1]->equals(*size_type))
-        throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-            "Second argument (size) for function {} must be Int64, but it has type {}",
-            name, argument_types[1]->getName());
-
-    if (argument_types.size() >= 3 && !argument_types[2]->equals(*ptr_type))
-        throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-            "Third argument (ptr) for function {} must be UInt64, but it has type {}",
-            name, argument_types[2]->getName());
-}
-
-AggregateFunctionPtr createAggregateFunctionFlameGraph(const std::string & name, const DataTypes & argument_types, const Array & params, const Settings * settings)
-{
-    if (!settings->allow_introspection_functions)
-        throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED,
-        "Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0");
-
-    check(name, argument_types, params);
-    return std::make_shared<AggregateFunctionFlameGraph>(argument_types);
-}
-
-void registerAggregateFunctionFlameGraph(AggregateFunctionFactory & factory)
-{
-    AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = true };
-
-    factory.registerFunction("flameGraph", { createAggregateFunctionFlameGraph, properties });
-}
-
-}

src/AggregateFunctions/AggregateFunctionForEach.h

@@ -107,7 +107,7 @@ private:
 
 public:
     AggregateFunctionForEach(AggregateFunctionPtr nested_, const DataTypes & arguments, const Array & params_)
-        : IAggregateFunctionDataHelper<AggregateFunctionForEachData, AggregateFunctionForEach>(arguments, params_)
+        : IAggregateFunctionDataHelper<AggregateFunctionForEachData, AggregateFunctionForEach>(arguments, params_, createResultType(nested_))
         , nested_func(nested_), num_arguments(arguments.size())
     {
         nested_size_of_data = nested_func->sizeOfData();
@@ -125,9 +125,9 @@ public:
         return nested_func->getName() + "ForEach";
     }
 
-    DataTypePtr getReturnType() const override
+    static DataTypePtr createResultType(AggregateFunctionPtr nested_)
     {
-        return std::make_shared<DataTypeArray>(nested_func->getReturnType());
+        return std::make_shared<DataTypeArray>(nested_->getResultType());
     }
 
     bool isVersioned() const override

src/AggregateFunctions/AggregateFunctionGroupArray.h

@@ -121,7 +121,7 @@ public:
     explicit GroupArrayNumericImpl(
         const DataTypePtr & data_type_, const Array & parameters_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max(), UInt64 seed_ = 123456)
         : IAggregateFunctionDataHelper<GroupArrayNumericData<T, Trait::sampler != Sampler::NONE>, GroupArrayNumericImpl<T, Trait>>(
-            {data_type_}, parameters_)
+            {data_type_}, parameters_, std::make_shared<DataTypeArray>(data_type_))
         , max_elems(max_elems_)
         , seed(seed_)
     {
@@ -129,8 +129,6 @@ public:
 
     String getName() const override { return getNameByTrait<Trait>(); }
 
-    DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(this->argument_types[0]); }
-
     void insert(Data & a, const T & v, Arena * arena) const
     {
         ++a.total_values;
@@ -423,7 +421,7 @@ class GroupArrayGeneralImpl final
 public:
     GroupArrayGeneralImpl(const DataTypePtr & data_type_, const Array & parameters_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max(), UInt64 seed_ = 123456)
         : IAggregateFunctionDataHelper<GroupArrayGeneralData<Node, Trait::sampler != Sampler::NONE>, GroupArrayGeneralImpl<Node, Trait>>(
-            {data_type_}, parameters_)
+            {data_type_}, parameters_, std::make_shared<DataTypeArray>(data_type_))
         , data_type(this->argument_types[0])
         , max_elems(max_elems_)
         , seed(seed_)
@@ -432,8 +430,6 @@ public:
 
     String getName() const override { return getNameByTrait<Trait>(); }
 
-    DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(data_type); }
-
     void insert(Data & a, const Node * v, Arena * arena) const
     {
         ++a.total_values;
@@ -697,7 +693,7 @@ class GroupArrayGeneralListImpl final
 
 public:
     GroupArrayGeneralListImpl(const DataTypePtr & data_type_, const Array & parameters_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
-        : IAggregateFunctionDataHelper<GroupArrayGeneralListData<Node>, GroupArrayGeneralListImpl<Node, Trait>>({data_type_}, parameters_)
+        : IAggregateFunctionDataHelper<GroupArrayGeneralListData<Node>, GroupArrayGeneralListImpl<Node, Trait>>({data_type_}, parameters_, std::make_shared<DataTypeArray>(data_type_))
         , data_type(this->argument_types[0])
         , max_elems(max_elems_)
     {
@@ -705,8 +701,6 @@ public:
 
     String getName() const override { return getNameByTrait<Trait>(); }
 
-    DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(data_type); }
-
     void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override
     {
         if (limit_num_elems && data(place).elems >= max_elems)

src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h

@@ -64,7 +64,7 @@ private:
 
 public:
     AggregateFunctionGroupArrayInsertAtGeneric(const DataTypes & arguments, const Array & params)
-        : IAggregateFunctionDataHelper<AggregateFunctionGroupArrayInsertAtDataGeneric, AggregateFunctionGroupArrayInsertAtGeneric>(arguments, params)
+        : IAggregateFunctionDataHelper<AggregateFunctionGroupArrayInsertAtDataGeneric, AggregateFunctionGroupArrayInsertAtGeneric>(arguments, params, std::make_shared<DataTypeArray>(arguments[0]))
         , type(argument_types[0])
         , serialization(type->getDefaultSerialization())
     {
@@ -101,11 +101,6 @@ public:
 
     String getName() const override { return "groupArrayInsertAt"; }
 
-    DataTypePtr getReturnType() const override
-    {
-        return std::make_shared<DataTypeArray>(type);
-    }
-
     bool allocatesMemoryInArena() const override { return false; }
 
     void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override

src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h

@@ -93,12 +93,15 @@ public:
     using ColumnResult = ColumnVectorOrDecimal<ResultT>;
 
     explicit MovingImpl(const DataTypePtr & data_type_, UInt64 window_size_ = std::numeric_limits<UInt64>::max())
-        : IAggregateFunctionDataHelper<Data, MovingImpl<T, LimitNumElements, Data>>({data_type_}, {})
+        : IAggregateFunctionDataHelper<Data, MovingImpl<T, LimitNumElements, Data>>({data_type_}, {}, createResultType(data_type_))
         , window_size(window_size_) {}
 
     String getName() const override { return Data::name; }
 
-    DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(getReturnTypeElement()); }
+    static DataTypePtr createResultType(const DataTypePtr & argument)
+    {
+        return std::make_shared<DataTypeArray>(getReturnTypeElement(argument));
+    }
 
     void NO_SANITIZE_UNDEFINED add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override
     {
@@ -183,14 +186,14 @@ public:
     }
 
 private:
-    auto getReturnTypeElement() const
+    static auto getReturnTypeElement(const DataTypePtr & argument)
     {
         if constexpr (!is_decimal<ResultT>)
             return std::make_shared<DataTypeNumber<ResultT>>();
         else
         {
             using Res = DataTypeDecimal<ResultT>;
-            return std::make_shared<Res>(Res::maxPrecision(), getDecimalScale(*this->argument_types.at(0)));
+            return std::make_shared<Res>(Res::maxPrecision(), getDecimalScale(*argument));
         }
     }
 };

src/AggregateFunctions/AggregateFunctionGroupBitmap.h

@@ -19,13 +19,13 @@ class AggregateFunctionBitmap final : public IAggregateFunctionDataHelper<Data,
 {
 public:
     explicit AggregateFunctionBitmap(const DataTypePtr & type)
-        : IAggregateFunctionDataHelper<Data, AggregateFunctionBitmap<T, Data>>({type}, {})
+        : IAggregateFunctionDataHelper<Data, AggregateFunctionBitmap<T, Data>>({type}, {}, createResultType())
     {
     }
 
     String getName() const override { return Data::name(); }
 
-    DataTypePtr getReturnType() const override { return std::make_shared<DataTypeNumber<T>>(); }
+    static DataTypePtr createResultType() { return std::make_shared<DataTypeNumber<T>>(); }
 
     bool allocatesMemoryInArena() const override { return false; }
 
@@ -59,13 +59,13 @@ private:
     static constexpr size_t STATE_VERSION_1_MIN_REVISION = 54455;
 public:
     explicit AggregateFunctionBitmapL2(const DataTypePtr & type)
-        : IAggregateFunctionDataHelper<Data, AggregateFunctionBitmapL2<T, Data, Policy>>({type}, {})
+        : IAggregateFunctionDataHelper<Data, AggregateFunctionBitmapL2<T, Data, Policy>>({type}, {}, createResultType())
     {
     }
 
     String getName() const override { return Policy::name; }
 
-    DataTypePtr getReturnType() const override { return std::make_shared<DataTypeNumber<T>>(); }
+    static DataTypePtr createResultType() { return std::make_shared<DataTypeNumber<T>>(); }
 
     bool allocatesMemoryInArena() const override { return false; }
 

src/AggregateFunctions/AggregateFunctionGroupUniqArray.cpp

@@ -26,8 +26,8 @@ class AggregateFunctionGroupUniqArrayDate : public AggregateFunctionGroupUniqArr
 {
 public:
     explicit AggregateFunctionGroupUniqArrayDate(const DataTypePtr & argument_type, const Array & parameters_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
-        : AggregateFunctionGroupUniqArray<DataTypeDate::FieldType, HasLimit>(argument_type, parameters_, max_elems_) {}
+        : AggregateFunctionGroupUniqArray<DataTypeDate::FieldType, HasLimit>(argument_type, parameters_, createResultType(), max_elems_) {}
-    DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(std::make_shared<DataTypeDate>()); }
+    static DataTypePtr createResultType() { return std::make_shared<DataTypeArray>(std::make_shared<DataTypeDate>()); }
 };
 
 template <typename HasLimit>
@@ -35,8 +35,8 @@ class AggregateFunctionGroupUniqArrayDateTime : public AggregateFunctionGroupUni
 {
 public:
     explicit AggregateFunctionGroupUniqArrayDateTime(const DataTypePtr & argument_type, const Array & parameters_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
-        : AggregateFunctionGroupUniqArray<DataTypeDateTime::FieldType, HasLimit>(argument_type, parameters_, max_elems_) {}
+        : AggregateFunctionGroupUniqArray<DataTypeDateTime::FieldType, HasLimit>(argument_type, parameters_, createResultType(), max_elems_) {}
-    DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(std::make_shared<DataTypeDateTime>()); }
+    static DataTypePtr createResultType() { return std::make_shared<DataTypeArray>(std::make_shared<DataTypeDateTime>()); }
 };
 
 template <typename HasLimit, typename ... TArgs>

src/AggregateFunctions/AggregateFunctionGroupUniqArray.h

@@ -50,15 +50,16 @@ private:
 public:
     AggregateFunctionGroupUniqArray(const DataTypePtr & argument_type, const Array & parameters_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
         : IAggregateFunctionDataHelper<AggregateFunctionGroupUniqArrayData<T>,
-          AggregateFunctionGroupUniqArray<T, LimitNumElems>>({argument_type}, parameters_),
+          AggregateFunctionGroupUniqArray<T, LimitNumElems>>({argument_type}, parameters_, std::make_shared<DataTypeArray>(argument_type)),
           max_elems(max_elems_) {}
 
-    String getName() const override { return "groupUniqArray"; }
+    AggregateFunctionGroupUniqArray(const DataTypePtr & argument_type, const Array & parameters_, const DataTypePtr & result_type_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
+        : IAggregateFunctionDataHelper<AggregateFunctionGroupUniqArrayData<T>,
+          AggregateFunctionGroupUniqArray<T, LimitNumElems>>({argument_type}, parameters_, result_type_),
+          max_elems(max_elems_) {}
 
-    DataTypePtr getReturnType() const override
+
-    {
+    String getName() const override { return "groupUniqArray"; }
-        return std::make_shared<DataTypeArray>(this->argument_types[0]);
-    }
 
     bool allocatesMemoryInArena() const override { return false; }
 
@@ -153,17 +154,12 @@ class AggregateFunctionGroupUniqArrayGeneric
 
 public:
     AggregateFunctionGroupUniqArrayGeneric(const DataTypePtr & input_data_type_, const Array & parameters_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
-        : IAggregateFunctionDataHelper<AggregateFunctionGroupUniqArrayGenericData, AggregateFunctionGroupUniqArrayGeneric<is_plain_column, LimitNumElems>>({input_data_type_}, parameters_)
+        : IAggregateFunctionDataHelper<AggregateFunctionGroupUniqArrayGenericData, AggregateFunctionGroupUniqArrayGeneric<is_plain_column, LimitNumElems>>({input_data_type_}, parameters_, std::make_shared<DataTypeArray>(input_data_type_))
         , input_data_type(this->argument_types[0])
         , max_elems(max_elems_) {}
 
     String getName() const override { return "groupUniqArray"; }
 
-    DataTypePtr getReturnType() const override
-    {
-        return std::make_shared<DataTypeArray>(input_data_type);
-    }
-
     bool allocatesMemoryInArena() const override
     {
         return true;

src/AggregateFunctions/AggregateFunctionHistogram.h

@@ -307,7 +307,7 @@ private:
 
 public:
     AggregateFunctionHistogram(UInt32 max_bins_, const DataTypes & arguments, const Array & params)
-        : IAggregateFunctionDataHelper<AggregateFunctionHistogramData, AggregateFunctionHistogram<T>>(arguments, params)
+        : IAggregateFunctionDataHelper<AggregateFunctionHistogramData, AggregateFunctionHistogram<T>>(arguments, params, createResultType())
         , max_bins(max_bins_)
     {
     }
@@ -316,7 +316,7 @@ public:
     {
         return Data::structSize(max_bins);
     }
-    DataTypePtr getReturnType() const override
+    static DataTypePtr createResultType()
     {
         DataTypes types;
         auto mean = std::make_shared<DataTypeNumber<Data::Mean>>();

src/AggregateFunctions/AggregateFunctionIf.cpp

@@ -23,7 +23,7 @@ public:
             throw Exception("Incorrect number of arguments for aggregate function with " + getName() + " suffix",
                 ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
 
-        if (!isUInt8(arguments.back()))
+        if (!isUInt8(arguments.back()) && !arguments.back()->onlyNull())
             throw Exception("Illegal type " + arguments.back()->getName() + " of last argument for aggregate function with " + getName() + " suffix",
                 ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
 
@@ -52,6 +52,7 @@ class AggregateFunctionIfNullUnary final
 private:
     size_t num_arguments;
     bool filter_is_nullable = false;
+    bool filter_is_only_null = false;
 
     /// The name of the nested function, including combinators (i.e. *If)
     ///
@@ -84,10 +85,8 @@ private:
 
             return assert_cast<const ColumnUInt8 &>(*filter_column).getData()[row_num] && !filter_null_map[row_num];
         }
-        else
+
-        {
+        return assert_cast<const ColumnUInt8 &>(*filter_column).getData()[row_num];
-            return assert_cast<const ColumnUInt8 &>(*filter_column).getData()[row_num];
-        }
     }
 
 public:
@@ -106,10 +105,14 @@ public:
                 "Aggregate function {} require at least one argument", getName());
 
         filter_is_nullable = arguments[num_arguments - 1]->isNullable();
+        filter_is_only_null = arguments[num_arguments - 1]->onlyNull();
     }
 
     void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override
     {
+        if (filter_is_only_null)
+            return;
+
         const ColumnNullable * column = assert_cast<const ColumnNullable *>(columns[0]);
         const IColumn * nested_column = &column->getNestedColumn();
         if (!column->isNullAt(row_num) && singleFilter(columns, row_num))
@@ -127,6 +130,9 @@ public:
         Arena * arena,
         ssize_t) const override
     {
+        if (filter_is_only_null)
+            return;
+
         const ColumnNullable * column = assert_cast<const ColumnNullable *>(columns[0]);
         const UInt8 * null_map = column->getNullMapData().data();
         const IColumn * columns_param[] = {&column->getNestedColumn()};
@@ -177,6 +183,11 @@ public:
 
 #if USE_EMBEDDED_COMPILER
 
+    bool isCompilable() const override
+    {
+        return canBeNativeType(*this->argument_types.back()) && this->nested_function->isCompilable();
+    }
+
     void compileAdd(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const DataTypes & arguments_types, const std::vector<llvm::Value *> & argument_values) const override
     {
         llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
@@ -224,6 +235,9 @@ class AggregateFunctionIfNullVariadic final : public AggregateFunctionNullBase<
                                                   serialize_flag,
                                                   AggregateFunctionIfNullVariadic<result_is_nullable, serialize_flag>>
 {
+private:
+    bool filter_is_only_null = false;
+
 public:
 
     String getName() const override
@@ -243,6 +257,8 @@ public:
 
         for (size_t i = 0; i < number_of_arguments; ++i)
             is_nullable[i] = arguments[i]->isNullable();
+
+        filter_is_only_null = arguments.back()->onlyNull();
     }
 
     static inline bool singleFilter(const IColumn ** columns, size_t row_num, size_t num_arguments)
@@ -282,6 +298,9 @@ public:
     void addBatchSinglePlace(
         size_t row_begin, size_t row_end, AggregateDataPtr __restrict place, const IColumn ** columns, Arena * arena, ssize_t) const final
     {
+        if (filter_is_only_null)
+            return;
+
         std::unique_ptr<UInt8[]> final_null_flags = std::make_unique<UInt8[]>(row_end);
         const size_t filter_column_num = number_of_arguments - 1;
 
@@ -346,6 +365,11 @@ public:
 
 #if USE_EMBEDDED_COMPILER
 
+    bool isCompilable() const override
+    {
+        return canBeNativeType(*this->argument_types.back()) && this->nested_function->isCompilable();
+    }
+
     void compileAdd(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const DataTypes & arguments_types, const std::vector<llvm::Value *> & argument_values) const override
     {
         /// TODO: Check
@@ -448,7 +472,7 @@ AggregateFunctionPtr AggregateFunctionIf::getOwnNullAdapter(
 
     /// Nullability of the last argument (condition) does not affect the nullability of the result (NULL is processed as false).
     /// For other arguments it is as usual (at least one is NULL then the result is NULL if possible).
-    bool return_type_is_nullable = !properties.returns_default_when_only_null && getReturnType()->canBeInsideNullable()
+    bool return_type_is_nullable = !properties.returns_default_when_only_null && getResultType()->canBeInsideNullable()
         && std::any_of(arguments.begin(), arguments.end() - 1, [](const auto & element) { return element->isNullable(); });
 
     bool need_to_serialize_flag = return_type_is_nullable || properties.returns_default_when_only_null;

src/AggregateFunctions/AggregateFunctionIf.h

@@ -36,13 +36,13 @@ private:
 
 public:
     AggregateFunctionIf(AggregateFunctionPtr nested, const DataTypes & types, const Array & params_)
-        : IAggregateFunctionHelper<AggregateFunctionIf>(types, params_)
+        : IAggregateFunctionHelper<AggregateFunctionIf>(types, params_, nested->getResultType())
         , nested_func(nested), num_arguments(types.size())
     {
         if (num_arguments == 0)
             throw Exception("Aggregate function " + getName() + " require at least one argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
 
-        if (!isUInt8(types.back()))
+        if (!isUInt8(types.back()) && !types.back()->onlyNull())
             throw Exception("Last argument for aggregate function " + getName() + " must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
     }
 
@@ -51,11 +51,6 @@ public:
         return nested_func->getName() + "If";
     }
 
-    DataTypePtr getReturnType() const override
-    {
-        return nested_func->getReturnType();
-    }
-
     const IAggregateFunction & getBaseAggregateFunctionWithSameStateRepresentation() const override
     {
         return nested_func->getBaseAggregateFunctionWithSameStateRepresentation();
@@ -204,12 +199,16 @@ public:
 
     AggregateFunctionPtr getNestedFunction() const override { return nested_func; }
 
+    std::unordered_set<size_t> getArgumentsThatCanBeOnlyNull() const override
+    {
+        return {num_arguments - 1};
+    }
 
 #if USE_EMBEDDED_COMPILER
 
     bool isCompilable() const override
     {
-        return nested_func->isCompilable();
+        return canBeNativeType(*this->argument_types.back()) && nested_func->isCompilable();
     }
 
     void compileCreate(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr) const override

src/AggregateFunctions/AggregateFunctionIntervalLengthSum.h

@@ -177,11 +177,11 @@ public:
     String getName() const override { return "intervalLengthSum"; }
 
     explicit AggregateFunctionIntervalLengthSum(const DataTypes & arguments)
-        : IAggregateFunctionDataHelper<Data, AggregateFunctionIntervalLengthSum<T, Data>>(arguments, {})
+        : IAggregateFunctionDataHelper<Data, AggregateFunctionIntervalLengthSum<T, Data>>(arguments, {}, createResultType())
     {
     }
 
-    DataTypePtr getReturnType() const override
+    static DataTypePtr createResultType()
     {
         if constexpr (std::is_floating_point_v<T>)
             return std::make_shared<DataTypeFloat64>();