Merge branch 'master' into enable-memory-overcommit

.clang-tidy

@@ -164,6 +164,8 @@ Checks: '-*,
     clang-analyzer-unix.cstring.NullArg,
 
     boost-use-to-string,
+
+    alpha.security.cert.env.InvalidPtr,
 '
 WarningsAsErrors: '*'
 

.github/workflows/backport_branches.yml

@@ -9,6 +9,18 @@ on: # yamllint disable-line rule:truthy
     branches:
       - 'backport/**'
 jobs:
+  PythonUnitTests:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Python unit tests
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 -m unittest discover -s . -p '*_test.py'
   DockerHubPushAarch64:
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
@@ -143,8 +155,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -184,8 +196,8 @@ jobs:
       - name: Upload build URLs to artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -229,8 +241,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -274,8 +286,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -319,8 +331,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -341,10 +353,15 @@ jobs:
     steps:
       - name: Set envs
         run: |
+          DEPENDENCIES=$(cat << 'EOF' | jq '. | length'
+          ${{ toJSON(needs) }}
+          EOF
+          )
+          echo "DEPENDENCIES=$DEPENDENCIES" >> "$GITHUB_ENV"
           cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/report_check
-          REPORTS_PATH=${{runner.temp}}/reports_dir
           CHECK_NAME=ClickHouse build check (actions)
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          TEMP_PATH=${{runner.temp}}/report_check
           EOF
       - name: Download json reports
         uses: actions/download-artifact@v2
@@ -360,7 +377,7 @@ jobs:
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 build_report_check.py "$CHECK_NAME"
+          python3 build_report_check.py "$CHECK_NAME" "$DEPENDENCIES"
       - name: Cleanup
         if: always()
         run: |

.github/workflows/master.yml

@@ -219,8 +219,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -260,8 +260,8 @@ jobs:
       - name: Upload build URLs to artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -305,8 +305,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -350,8 +350,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -395,8 +395,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -440,8 +440,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -485,8 +485,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -530,8 +530,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -575,8 +575,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -620,8 +620,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -668,8 +668,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -713,8 +713,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -758,8 +758,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -803,8 +803,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -848,8 +848,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -893,8 +893,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -938,8 +938,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -992,10 +992,16 @@ jobs:
     steps:
       - name: Set envs
         run: |
+          DEPENDENCIES=$(cat << 'EOF' | jq '. | length'
+          ${{ toJSON(needs) }}
+          EOF
+          )
+          echo "DEPENDENCIES=$DEPENDENCIES" >> "$GITHUB_ENV"
           cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/report_check
-          REPORTS_PATH=${{runner.temp}}/reports_dir
           CHECK_NAME=ClickHouse build check (actions)
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          TEMP_PATH=${{runner.temp}}/report_check
           EOF
       - name: Download json reports
         uses: actions/download-artifact@v2
@@ -1011,7 +1017,7 @@ jobs:
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 build_report_check.py "$CHECK_NAME"
+          python3 build_report_check.py "$CHECK_NAME" "$DEPENDENCIES"
       - name: Cleanup
         if: always()
         run: |

.github/workflows/nightly.yml

@@ -72,3 +72,53 @@ jobs:
         with:
           name: changed_images
           path: ${{ runner.temp }}/changed_images.json
+  BuilderCoverity:
+    needs: DockerHubPush
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          BUILD_NAME=coverity
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          CHECK_NAME=ClickHouse build check (actions)
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          TEMP_PATH=${{runner.temp}}/build_check
+          EOF
+          echo "COVERITY_TOKEN=${{ secrets.COVERITY_TOKEN }}" >> "$GITHUB_ENV"
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        id: coverity-checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: 'true'
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME"
+      - name: Upload Coverity Analysis
+        if: ${{ success() || failure() }}
+        run: |
+          curl --form token="${COVERITY_TOKEN}" \
+            --form email='security+coverity@clickhouse.com' \
+            --form file="@$TEMP_PATH/$BUILD_NAME/coverity-scan.tgz" \
+            --form version="${GITHUB_REF#refs/heads/}-${GITHUB_SHA::6}" \
+            --form description="Nighly Scan: $(date +'%Y-%m-%dT%H:%M:%S')" \
+            https://scan.coverity.com/builds?project=ClickHouse%2FClickHouse
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"

.github/workflows/pull_request.yml

@@ -272,8 +272,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -317,8 +317,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -362,8 +362,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -404,8 +404,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -446,8 +446,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -491,8 +491,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -536,8 +536,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -581,8 +581,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -626,8 +626,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -671,8 +671,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -719,8 +719,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -764,8 +764,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -809,8 +809,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -854,8 +854,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -899,8 +899,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -944,8 +944,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -989,8 +989,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -1044,10 +1044,16 @@ jobs:
     steps:
       - name: Set envs
         run: |
+          DEPENDENCIES=$(cat << 'EOF' | jq '. | length'
+          ${{ toJSON(needs) }}
+          EOF
+          )
+          echo "DEPENDENCIES=$DEPENDENCIES" >> "$GITHUB_ENV"
           cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/report_check
-          REPORTS_PATH=${{runner.temp}}/reports_dir
           CHECK_NAME=ClickHouse build check (actions)
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          TEMP_PATH=${{runner.temp}}/report_check
           EOF
       - name: Download json reports
         uses: actions/download-artifact@v2
@@ -1063,7 +1069,7 @@ jobs:
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 build_report_check.py "$CHECK_NAME"
+          python3 build_report_check.py "$CHECK_NAME" "$DEPENDENCIES"
       - name: Cleanup
         if: always()
         run: |

.github/workflows/release.yml

@@ -52,8 +52,8 @@ jobs:
     - name: Check docker clickhouse/clickhouse-server building
       run: |
         cd "$GITHUB_WORKSPACE/tests/ci"
-        python3 docker_server.py --release-type auto
-        python3 docker_server.py --release-type auto --no-ubuntu \
+        python3 docker_server.py --release-type auto --version "${{ github.ref }}"
+        python3 docker_server.py --release-type auto --version "${{ github.ref }}" --no-ubuntu \
           --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
     - name: Cleanup
       if: always()

.github/workflows/release_branches.yml

@@ -146,8 +146,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -187,8 +187,8 @@ jobs:
       - name: Upload build URLs to artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -232,8 +232,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -277,8 +277,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -322,8 +322,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -367,8 +367,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -412,8 +412,8 @@ jobs:
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.BUILD_NAME }}
-          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
       - name: Cleanup
         if: always()
         run: |
@@ -436,10 +436,16 @@ jobs:
     steps:
       - name: Set envs
         run: |
+          DEPENDENCIES=$(cat << 'EOF' | jq '. | length'
+          ${{ toJSON(needs) }}
+          EOF
+          )
+          echo "DEPENDENCIES=$DEPENDENCIES" >> "$GITHUB_ENV"
           cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/report_check
-          REPORTS_PATH=${{runner.temp}}/reports_dir
           CHECK_NAME=ClickHouse build check (actions)
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          TEMP_PATH=${{runner.temp}}/report_check
           EOF
       - name: Download json reports
         uses: actions/download-artifact@v2
@@ -455,7 +461,7 @@ jobs:
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 build_report_check.py "$CHECK_NAME"
+          python3 build_report_check.py "$CHECK_NAME" "$DEPENDENCIES"
       - name: Cleanup
         if: always()
         run: |

CMakeLists.txt

@@ -222,6 +222,12 @@ else ()
     set(NO_WHOLE_ARCHIVE --no-whole-archive)
 endif ()
 
+option(ENABLE_CURL_BUILD "Enable curl, azure, sentry build on by default except MacOS." ON)
+if (OS_DARWIN)
+    # Disable the curl, azure, senry build on MacOS
+    set (ENABLE_CURL_BUILD OFF)
+endif ()
+
 # Ignored if `lld` is used
 option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker.")
 
@@ -294,14 +300,19 @@ include(cmake/cpu_features.cmake)
 # Enable it explicitly.
 set (COMPILER_FLAGS "${COMPILER_FLAGS} -fasynchronous-unwind-tables")
 
-# Reproducible builds
-# If turned `ON`, remap file source paths in debug info, predefined preprocessor macros and __builtin_FILE().
-option(ENABLE_BUILD_PATH_MAPPING "Enable remap file source paths in debug info, predefined preprocessor macros and __builtin_FILE(). It's to generate reproducible builds. See https://reproducible-builds.org/docs/build-path" ON)
+# Reproducible builds.
+if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
+    set (ENABLE_BUILD_PATH_MAPPING_DEFAULT OFF)
+else ()
+    set (ENABLE_BUILD_PATH_MAPPING_DEFAULT ON)
+endif ()
+
+option (ENABLE_BUILD_PATH_MAPPING "Enable remapping of file source paths in debug info, predefined preprocessor macros, and __builtin_FILE(). It's used to generate reproducible builds. See https://reproducible-builds.org/docs/build-path" ${ENABLE_BUILD_PATH_MAPPING_DEFAULT})
 
 if (ENABLE_BUILD_PATH_MAPPING)
     set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
     set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
-endif()
+endif ()
 
 if (${CMAKE_VERSION} VERSION_LESS "3.12.4")
     # CMake < 3.12 doesn't support setting 20 as a C++ standard version.

base/daemon/BaseDaemon.cpp

@@ -828,7 +828,6 @@ void BaseDaemon::initializeTerminationAndSignalProcessing()
 
     /// Setup signal handlers.
     /// SIGTSTP is added for debugging purposes. To output a stack trace of any running thread at anytime.
-
     addSignalHandler({SIGABRT, SIGSEGV, SIGILL, SIGBUS, SIGSYS, SIGFPE, SIGPIPE, SIGTSTP, SIGTRAP}, signalHandler, &handled_signals);
     addSignalHandler({SIGHUP}, closeLogsSignalHandler, &handled_signals);
     addSignalHandler({SIGINT, SIGQUIT, SIGTERM}, terminateRequestedSignalHandler, &handled_signals);

base/loggers/Loggers.cpp

@@ -197,7 +197,6 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
 
         Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(color_enabled);
         Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, new Poco::ConsoleChannel);
-        logger.warning("Logging " + console_log_level_string + " to console");
         log->setLevel(console_log_level);
         split->addChannel(log, "console");
     }

contrib/CMakeLists.txt

@@ -119,9 +119,13 @@ add_contrib (fastops-cmake fastops)
 add_contrib (libuv-cmake libuv)
 add_contrib (amqpcpp-cmake AMQP-CPP) # requires: libuv
 add_contrib (cassandra-cmake cassandra) # requires: libuv
-add_contrib (curl-cmake curl)
-add_contrib (azure-cmake azure)
-add_contrib (sentry-native-cmake sentry-native) # requires: curl
+
+if (ENABLE_CURL_BUILD)
+    add_contrib (curl-cmake curl)
+    add_contrib (azure-cmake azure)
+    add_contrib (sentry-native-cmake sentry-native) # requires: curl
+endif()
+
 add_contrib (fmtlib-cmake fmtlib)
 add_contrib (krb5-cmake krb5)
 add_contrib (cyrus-sasl-cmake cyrus-sasl) # for krb5

contrib/arrow

@@ -1 +1 @@
-Subproject commit 1d9cc51daa4e7e9fc6926320ef73759818bd736e
+Subproject commit efdcd015cfdee1b6aa349c9ca227ca12c3d697f5

contrib/curl

@@ -1 +1 @@
-Subproject commit 3b8bbbbd1609c638a3d3d0acb148a33dedb67be3
+Subproject commit 801bd5138ce31aa0d906fa4e2eabfc599d74e793

contrib/curl-cmake/CMakeLists.txt

@@ -32,7 +32,6 @@ set (SRCS
     "${LIBRARY_DIR}/lib/transfer.c"
     "${LIBRARY_DIR}/lib/strcase.c"
     "${LIBRARY_DIR}/lib/easy.c"
-    "${LIBRARY_DIR}/lib/security.c"
     "${LIBRARY_DIR}/lib/curl_fnmatch.c"
     "${LIBRARY_DIR}/lib/fileinfo.c"
     "${LIBRARY_DIR}/lib/wildcard.c"
@@ -115,6 +114,12 @@ set (SRCS
     "${LIBRARY_DIR}/lib/curl_get_line.c"
     "${LIBRARY_DIR}/lib/altsvc.c"
     "${LIBRARY_DIR}/lib/socketpair.c"
+    "${LIBRARY_DIR}/lib/bufref.c"
+    "${LIBRARY_DIR}/lib/dynbuf.c"
+    "${LIBRARY_DIR}/lib/hsts.c"
+    "${LIBRARY_DIR}/lib/http_aws_sigv4.c"
+    "${LIBRARY_DIR}/lib/mqtt.c"
+    "${LIBRARY_DIR}/lib/rename.c"
     "${LIBRARY_DIR}/lib/vauth/vauth.c"
     "${LIBRARY_DIR}/lib/vauth/cleartext.c"
     "${LIBRARY_DIR}/lib/vauth/cram.c"
@@ -131,8 +136,6 @@ set (SRCS
     "${LIBRARY_DIR}/lib/vtls/gtls.c"
     "${LIBRARY_DIR}/lib/vtls/vtls.c"
     "${LIBRARY_DIR}/lib/vtls/nss.c"
-    "${LIBRARY_DIR}/lib/vtls/polarssl.c"
-    "${LIBRARY_DIR}/lib/vtls/polarssl_threadlock.c"
     "${LIBRARY_DIR}/lib/vtls/wolfssl.c"
     "${LIBRARY_DIR}/lib/vtls/schannel.c"
     "${LIBRARY_DIR}/lib/vtls/schannel_verify.c"
@@ -141,6 +144,7 @@ set (SRCS
     "${LIBRARY_DIR}/lib/vtls/mbedtls.c"
     "${LIBRARY_DIR}/lib/vtls/mesalink.c"
     "${LIBRARY_DIR}/lib/vtls/bearssl.c"
+    "${LIBRARY_DIR}/lib/vtls/keylog.c"
     "${LIBRARY_DIR}/lib/vquic/ngtcp2.c"
     "${LIBRARY_DIR}/lib/vquic/quiche.c"
     "${LIBRARY_DIR}/lib/vssh/libssh2.c"

contrib/replxx

@@ -1 +1 @@
-Subproject commit 6f0b6f151ae2a044625ae93acd19ca365fcea64d
+Subproject commit 3fd0e3c9364a589447453d9906d854ebd8d385c5

docker/docs/builder/Dockerfile

@@ -1,4 +1,3 @@
-# rebuild in #33610
 # docker build -t clickhouse/docs-builder .
 FROM ubuntu:20.04
 

docker/packager/binary/build.sh

@@ -25,13 +25,23 @@ read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}"
 env
 cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
 
+if [ "coverity" == "$COMBINED_OUTPUT" ]
+then
+    mkdir -p /opt/cov-analysis
+
+    wget --post-data "token=$COVERITY_TOKEN&project=ClickHouse%2FClickHouse" -qO- https://scan.coverity.com/download/linux64 | tar xz -C /opt/cov-analysis --strip-components 1
+    export PATH=$PATH:/opt/cov-analysis/bin
+    cov-configure --config ./coverity.config --template --comptype clangcc --compiler "$CC"
+    SCAN_WRAPPER="cov-build --config ./coverity.config --dir cov-int"
+fi
+
 cache_status
 # clear cache stats
 ccache --zero-stats ||:
 
 # No quotes because I want it to expand to nothing if empty.
-# shellcheck disable=SC2086
-ninja $NINJA_FLAGS clickhouse-bundle
+# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
+$SCAN_WRAPPER ninja $NINJA_FLAGS clickhouse-bundle
 
 cache_status
 
@@ -91,6 +101,12 @@ then
     mv "$COMBINED_OUTPUT.tgz" /output
 fi
 
+if [ "coverity" == "$COMBINED_OUTPUT" ]
+then
+    tar -cv -I pigz -f "coverity-scan.tgz" cov-int
+    mv "coverity-scan.tgz" /output
+fi
+
 # Also build fuzzers if any sanitizer specified
 # if [ -n "$SANITIZER" ]
 # then

docker/packager/packager

@@ -176,6 +176,9 @@ def parse_env_variables(
     if package_type == "performance":
         result.append("COMBINED_OUTPUT=performance")
         cmake_flags.append("-DENABLE_TESTS=0")
+    elif package_type == "coverity":
+        result.append("COMBINED_OUTPUT=coverity")
+        result.append('COVERITY_TOKEN="$COVERITY_TOKEN"')
     elif split_binary:
         result.append("COMBINED_OUTPUT=shared_build")
 
@@ -262,9 +265,8 @@ if __name__ == "__main__":
     # and configs to be used for performance test.
     parser.add_argument(
         "--package-type",
-        choices=("deb", "binary", "performance"),
+        choices=["deb", "binary", "performance", "coverity"],
         required=True,
-        help="a build type",
     )
     parser.add_argument(
         "--clickhouse-repo-path",
@@ -330,7 +332,11 @@ if __name__ == "__main__":
     if not os.path.isabs(args.output_dir):
         args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir))
 
-    image_type = "binary" if args.package_type == "performance" else args.package_type
+    image_type = (
+        "binary"
+        if args.package_type in ("performance", "coverity")
+        else args.package_type
+    )
     image_name = "clickhouse/binary-builder"
 
     if not os.path.isabs(args.clickhouse_repo_path):

docker/test/fuzzer/run-fuzzer.sh

@@ -226,7 +226,6 @@ quit
         --receive_data_timeout_ms=10000 \
         --stacktrace \
         --query-fuzzer-runs=1000 \
-        --testmode \
         --queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) \
         $NEW_TESTS_OPT \
         > >(tail -n 100000 > fuzzer.log) \

docker/test/integration/mysql_js_client/Dockerfile

@@ -1,8 +1,10 @@
 # docker build -t clickhouse/mysql-js-client .
 # MySQL JavaScript client docker container
 
-FROM node:8
+FROM node:16.14.2
+
+WORKDIR /usr/app
 
 RUN npm install mysql
 
-COPY ./test.js test.js
+COPY ./test.js ./test.js

docker/test/stateful/run.sh

@@ -96,7 +96,7 @@ else
     clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
     clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
     clickhouse-client --query "CREATE TABLE test.hits_s3  (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
-    clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM test.hits"
+    clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM test.hits SETTINGS enable_filesystem_cache_on_write_operations=0"
 fi
 
 clickhouse-client --query "SHOW TABLES FROM test"

docker/test/stateless/run.sh

@@ -139,7 +139,7 @@ pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhous
 #   directly
 # - even though ci auto-compress some files (but not *.tsv) it does this only
 #   for files >64MB, we want this files to be compressed explicitly
-for table in query_log zookeeper_log trace_log
+for table in query_log zookeeper_log trace_log transactions_info_log
 do
     clickhouse-client -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.tsv.gz &
     if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then

docker/test/stress/run.sh

@@ -106,17 +106,6 @@ function stop()
 
 function start()
 {
-    # Rename existing log file - it will be more convenient to read separate files for separate server runs.
-    if [ -f '/var/log/clickhouse-server/clickhouse-server.log' ]
-    then
-        log_file_counter=1
-        while [ -f "/var/log/clickhouse-server/clickhouse-server.log.${log_file_counter}" ]
-        do
-            log_file_counter=$((log_file_counter + 1))
-        done
-        mv '/var/log/clickhouse-server/clickhouse-server.log' "/var/log/clickhouse-server/clickhouse-server.log.${log_file_counter}"
-    fi
-
     counter=0
     until clickhouse-client --query "SELECT 1"
     do
@@ -190,6 +179,8 @@ clickhouse-client --query "ATTACH DATABASE IF NOT EXISTS datasets ENGINE = Ordin
 clickhouse-client --query "CREATE DATABASE IF NOT EXISTS test"
 
 stop
+mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.initial.log
+
 start
 
 clickhouse-client --query "SHOW TABLES FROM datasets"
@@ -205,6 +196,8 @@ clickhouse-client --query "SHOW TABLES FROM test"
     || echo -e 'Test script failed\tFAIL' >> /test_output/test_results.tsv
 
 stop
+mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.stress.log
+
 start
 
 clickhouse-client --query "SELECT 'Server successfully started', 'OK'" >> /test_output/test_results.tsv \
@@ -263,10 +256,12 @@ mkdir previous_release_package_folder
 clickhouse-client --query="SELECT version()" | ./download_previous_release && echo -e 'Download script exit code\tOK' >> /test_output/test_results.tsv \
     || echo -e 'Download script failed\tFAIL' >> /test_output/test_results.tsv
 
+stop
+mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.clean.log
+
 if [ "$(ls -A previous_release_package_folder/clickhouse-common-static_*.deb && ls -A previous_release_package_folder/clickhouse-server_*.deb)" ]
 then
     echo -e "Successfully downloaded previous release packets\tOK" >> /test_output/test_results.tsv
-    stop
 
     # Uninstall current packages
     dpkg --remove clickhouse-client
@@ -289,7 +284,7 @@ then
     install_packages package_folder
 
     mkdir tmp_stress_output
-    
+
     ./stress --backward-compatibility-check --output-folder tmp_stress_output --global-time-limit=1200 \
         && echo -e 'Backward compatibility check: Test script exit code\tOK' >> /test_output/test_results.tsv \
         || echo -e 'Backward compatibility check: Test script failed\tFAIL' >> /test_output/test_results.tsv
@@ -297,8 +292,9 @@ then
 
     clickhouse-client --query="SELECT 'Tables count:', count() FROM system.tables"
 
-    stop    
-    
+    stop
+    mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.stress.log
+
     # Start new server
     configure
     start 500
@@ -310,8 +306,9 @@ then
 
     # Let the server run for a while before checking log.
     sleep 60
-    
+
     stop
+    mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.clean.log
 
     # Error messages (we should ignore some errors)
     echo "Check for Error messages in server log:"
@@ -332,8 +329,8 @@ then
                -e "Code: 1000, e.code() = 111, Connection refused" \
                -e "UNFINISHED" \
                -e "Renaming unexpected part" \
-        /var/log/clickhouse-server/clickhouse-server.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
-        && echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
+        /var/log/clickhouse-server/clickhouse-server.backward.*.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
+        && echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tOK' >> /test_output/test_results.tsv \
         || echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
 
     # Remove file bc_check_error_messages.txt if it's empty
@@ -348,13 +345,13 @@ then
     rm -f /test_output/tmp
 
     # OOM
-    zgrep -Fa " <Fatal> Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \
-        && echo -e 'Backward compatibility check: OOM killer (or signal 9) in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \
+    zgrep -Fa " <Fatal> Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
+        && echo -e 'Backward compatibility check: OOM killer (or signal 9) in clickhouse-server.log\tOK' >> /test_output/test_results.tsv \
         || echo -e 'Backward compatibility check: No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
 
     # Logical errors
     echo "Check for Logical errors in server log:"
-    zgrep -Fa -A20 "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.log > /test_output/bc_check_logical_errors.txt \
+    zgrep -Fa -A20 "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_logical_errors.txt \
         && echo -e 'Backward compatibility check: Logical error thrown (see clickhouse-server.log or bc_check_logical_errors.txt)\tFAIL' >> /test_output/test_results.tsv \
         || echo -e 'Backward compatibility check: No logical errors\tOK' >> /test_output/test_results.tsv
 
@@ -362,19 +359,18 @@ then
     [ -s /test_output/bc_check_logical_errors.txt ] || rm /test_output/bc_check_logical_errors.txt
 
     # Crash
-    zgrep -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \
+    zgrep -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
         && echo -e 'Backward compatibility check: Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \
         || echo -e 'Backward compatibility check: Not crashed\tOK' >> /test_output/test_results.tsv
 
     # It also checks for crash without stacktrace (printed by watchdog)
     echo "Check for Fatal message in server log:"
-    zgrep -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server.log > /test_output/bc_check_fatal_messages.txt \
-        && echo -e 'Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
+    zgrep -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_fatal_messages.txt \
+        && echo -e 'Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)\tOK' >> /test_output/test_results.tsv \
         || echo -e 'Backward compatibility check: No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
 
     # Remove file bc_check_fatal_messages.txt if it's empty
     [ -s /test_output/bc_check_fatal_messages.txt ] || rm /test_output/bc_check_fatal_messages.txt
-
 else
     echo -e "Backward compatibility check: Failed to download previous release packets\tFAIL" >> /test_output/test_results.tsv
 fi

docker/test/stress/stress

@@ -10,7 +10,7 @@ import logging
 import time
 
 
-def get_options(i):
+def get_options(i, backward_compatibility_check):
     options = []
     client_options = []
     if 0 < i:
@@ -19,7 +19,7 @@ def get_options(i):
     if i % 3 == 1:
         options.append("--db-engine=Ordinary")
 
-    if i % 3 == 2:
+    if i % 3 == 2 and not backward_compatibility_check:
         options.append('''--db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i))
         client_options.append('allow_experimental_database_replicated=1')
 
@@ -57,7 +57,7 @@ def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_t
     pipes = []
     for i in range(0, len(output_paths)):
         f = open(output_paths[i], 'w')
-        full_command = "{} {} {} {} {}".format(cmd, get_options(i), global_time_limit_option, skip_tests_option, backward_compatibility_check_option)
+        full_command = "{} {} {} {} {}".format(cmd, get_options(i, backward_compatibility_check), global_time_limit_option, skip_tests_option, backward_compatibility_check_option)
         logging.info("Run func tests '%s'", full_command)
         p = Popen(full_command, shell=True, stdout=f, stderr=f)
         pipes.append(p)

docs/en/introduction/adopters.md

@@ -43,7 +43,7 @@ toc_title: Adopters
 | <a href="https://city-mobil.ru" class="favicon">Citymobil</a> | Taxi | Analytics | — | — | [Blog Post in Russian, March 2020](https://habr.com/en/company/citymobil/blog/490660/) |
 | <a href="https://cloudflare.com" class="favicon">Cloudflare</a> | CDN | Traffic analysis | 36 servers | — | [Blog post, May 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Blog post, March 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) |
 | <a href="https://corporate.comcast.com/" class="favicon">Comcast</a> | Media | CDN Traffic Analysis | — | — | [ApacheCon 2019 Talk](https://www.youtube.com/watch?v=e9TZ6gFDjNg) |
-| <a href="https://contentsquare.com" class="favicon">ContentSquare</a> | Web analytics | Main product | — | — | [Blog post in French, November 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) |
+| <a href="https://contentsquare.com" class="favicon">Contentsquare</a> | Web analytics | Main product | — | — | [Blog post in French, November 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) |
 | <a href="https://coru.net/" class="favicon">Corunet</a> | Analytics | Main product | — | — | [Slides in English, April 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) |
 | <a href="https://www.creditx.com" class="favicon">CraiditX 氪信</a> | Finance AI | Analysis | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) |
 | <a href="https://crazypanda.ru/en/" class="favicon">Crazypanda</a> | Games | | — | — | Live session on ClickHouse meetup |
@@ -158,6 +158,7 @@ toc_title: Adopters
 | <a href="https://www.staffcop.ru/" class="favicon">Staffcop</a> | Information Security | Main Product | — | — | [Official website, Documentation](https://www.staffcop.ru/sce43) |
 | <a href="https://www.suning.com/" class="favicon">Suning</a> | E-Commerce | User behaviour analytics | — | — | [Blog article](https://www.sohu.com/a/434152235_411876) |
 | <a href="https://superwall.me/" class="favicon">Superwall</a> | Monetization Tooling | Main product | — | — | [Word of mouth, Jan 2022](https://github.com/ClickHouse/ClickHouse/pull/33573) |
+| <a href="https://swetrix.com" class="favicon">Swetrix</a> | Analytics | Main Product | — | — | [Source code](https://github.com/swetrix/swetrix-api) |
 | <a href="https://www.teralytics.net/" class="favicon">Teralytics</a> | Mobility | Analytics | — | — | [Tech blog](https://www.teralytics.net/knowledge-hub/visualizing-mobility-data-the-scalability-challenge) |
 | <a href="https://www.tencent.com" class="favicon">Tencent</a> | Big Data | Data processing | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) |
 | <a href="https://www.tencent.com" class="favicon">Tencent</a> | Messaging | Logging | — | — | [Talk in Chinese, November 2019](https://youtu.be/T-iVQRuw-QY?t=5050) |

docs/en/operations/server-configuration-parameters/settings.md

@@ -1467,6 +1467,18 @@ The update is performed asynchronously, in a separate system thread.
 
 -   [background_schedule_pool_size](../../operations/settings/settings.md#background_schedule_pool_size)
 
+
+## dns_max_consecutive_failures {#server-settings-dns-max-consecutive-failures}
+
+The number of consecutive failures accepted when updating a DNS cache entry before it is dropped.
+Use `0` to disable cache dropping (entries will only be cleaned by `SYSTEM DROP DNS CACHE`)
+
+**Default value**: 5.
+
+**See also**
+
+-   [`SYSTEM DROP DNS CACHE`](../../sql-reference/statements/system.md#query_language-system-drop-dns-cache)
+
 ## distributed_ddl {#server-settings-distributed_ddl}
 
 Manage executing [distributed ddl queries](../../sql-reference/distributed-ddl.md)  (CREATE, DROP, ALTER, RENAME) on cluster.

docs/en/operations/settings/settings.md

@@ -519,6 +519,33 @@ Possible values:
 
 Default value: `1`.
 
+## allow_settings_after_format_in_insert {#allow_settings_after_format_in_insert}
+
+Control whether `SETTINGS` after `FORMAT` in `INSERT` queries is allowed or not. It is not recommended to use this, since this may interpret part of `SETTINGS` as values.
+
+Example:
+
+```sql
+INSERT INTO FUNCTION null('foo String') SETTINGS max_threads=1 VALUES ('bar');
+```
+
+But the following query will work only with `allow_settings_after_format_in_insert`:
+
+```sql
+SET allow_settings_after_format_in_insert=1;
+INSERT INTO FUNCTION null('foo String') VALUES ('bar') SETTINGS max_threads=1;
+```
+
+Possible values:
+
+-   0 — Disallow.
+-   1 — Allow.
+
+Default value: `0`.
+
+!!! note "Warning"
+    Use this setting only for backward compatibility if your use cases depend on old syntax.
+
 ## input_format_skip_unknown_fields {#settings-input-format-skip-unknown-fields}
 
 Enables or disables skipping insertion of extra data.
@@ -1062,6 +1089,15 @@ Result:
 └─────────────┴───────────┘
 ```
 
+## log_processors_profiles {#settings-log_processors_profiles}
+
+Write time that processor spent during execution/waiting for data to `system.processors_profile_log` table.
+
+See also:
+
+-   [`system.processors_profile_log`](../../operations/system-tables/processors_profile_log.md#system-processors_profile_log)
+-   [`EXPLAIN PIPELINE`](../../sql-reference/statements/explain.md#explain-pipeline)
+
 ## max_insert_block_size {#settings-max_insert_block_size}
 
 The size of blocks (in a count of rows) to form for insertion into a table.

docs/en/operations/system-tables/processors_profile_log.md

@@ -0,0 +1,75 @@
+# system.processors_profile_log {#system-processors_profile_log}
+
+This table contains profiling on processors level (that you can find in [`EXPLAIN PIPELINE`](../../sql-reference/statements/explain.md#explain-pipeline)).
+
+Columns:
+
+-   `event_date` ([Date](../../sql-reference/data-types/date.md)) — The date when the event happened.
+-   `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — The date and time when the event happened.
+-   `id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — ID of processor
+-   `parent_ids` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — Parent processors IDs
+-   `query_id` ([String](../../sql-reference/data-types/string.md)) — ID of the query
+-   `name` ([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md)) — Name of the processor.
+-   `elapsed_us` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of microseconds this processor was executed.
+-   `input_wait_elapsed_us` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of microseconds this processor was waiting for data (from other processor).
+-   `output_wait_elapsed_us` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of microseconds this processor was waiting because output port was full.
+
+**Example**
+
+Query:
+
+``` sql
+EXPLAIN PIPELINE
+SELECT sleep(1)
+
+┌─explain─────────────────────────┐
+│ (Expression)                    │
+│ ExpressionTransform             │
+│   (SettingQuotaAndLimits)       │
+│     (ReadFromStorage)           │
+│     SourceFromSingleChunk 0 → 1 │
+└─────────────────────────────────┘
+
+SELECT sleep(1)
+SETTINGS log_processors_profiles = 1
+
+Query id: feb5ed16-1c24-4227-aa54-78c02b3b27d4
+
+┌─sleep(1)─┐
+│        0 │
+└──────────┘
+
+1 rows in set. Elapsed: 1.018 sec.
+
+SELECT
+    name,
+    elapsed_us,
+    input_wait_elapsed_us,
+    output_wait_elapsed_us
+FROM system.processors_profile_log
+WHERE query_id = 'feb5ed16-1c24-4227-aa54-78c02b3b27d4'
+ORDER BY name ASC
+```
+
+Result:
+
+``` text
+┌─name────────────────────┬─elapsed_us─┬─input_wait_elapsed_us─┬─output_wait_elapsed_us─┐
+│ ExpressionTransform     │    1000497 │                  2823 │                    197 │
+│ LazyOutputFormat        │         36 │               1002188 │                      0 │
+│ LimitsCheckingTransform │         10 │               1002994 │                    106 │
+│ NullSource              │          5 │               1002074 │                      0 │
+│ NullSource              │          1 │               1002084 │                      0 │
+│ SourceFromSingleChunk   │         45 │                  4736 │                1000819 │
+└─────────────────────────┴────────────┴───────────────────────┴────────────────────────┘
+```
+
+Here you can see:
+
+-  `ExpressionTransform` was executing `sleep(1)` function, so it `work` will takes 1e6, and so `elapsed_us` > 1e6.
+-  `SourceFromSingleChunk` need to wait, because `ExpressionTransform` does not accept any data during execution of `sleep(1)`, so it will be in `PortFull` state for 1e6 us, and so `output_wait_elapsed_us` > 1e6.
+-  `LimitsCheckingTransform`/`NullSource`/`LazyOutputFormat` need to wait until `ExpressionTransform` will execute `sleep(1)` to process the result, so `input_wait_elapsed_us` > 1e6.
+
+**See Also**
+
+-   [`EXPLAIN PIPELINE`](../../sql-reference/statements/explain.md#explain-pipeline)

docs/en/sql-reference/statements/select/index.md

@@ -22,7 +22,7 @@ SELECT [DISTINCT [ON (column1, column2, ...)]] expr_list
 [WHERE expr]
 [GROUP BY expr_list] [WITH ROLLUP|WITH CUBE] [WITH TOTALS]
 [HAVING expr]
-[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr]
+[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr] [INTERPOLATE [(expr_list)]]
 [LIMIT [offset_value, ]n BY columns]
 [LIMIT [n, ]m] [WITH TIES]
 [SETTINGS ...]

docs/en/sql-reference/statements/select/order-by.md

@@ -280,6 +280,7 @@ To fill multiple columns, add `WITH FILL` modifier with optional parameters afte
 
 ``` sql
 ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr]
+[INTERPOLATE [(col [AS expr], ... colN [AS exprN])]]
 ```
 
 `WITH FILL` can be applied for fields with Numeric (all kinds of float, decimal, int) or Date/DateTime types. When applied for `String` fields, missed values are filled with empty strings.
@@ -287,6 +288,7 @@ When `FROM const_expr` not defined sequence of filling use minimal `expr` field
 When `TO const_expr` not defined sequence of filling use maximum `expr` field value from `ORDER BY`.
 When `STEP const_numeric_expr` defined then `const_numeric_expr` interprets `as is` for numeric types, as `days` for Date type, as `seconds` for DateTime type. It also supports [INTERVAL](https://clickhouse.com/docs/en/sql-reference/data-types/special-data-types/interval/) data type representing time and date intervals.
 When `STEP const_numeric_expr` omitted then sequence of filling use `1.0` for numeric type, `1 day` for Date type and `1 second` for DateTime type.
+`INTERPOLATE` can be applied to columns not participating in `ORDER BY WITH FILL`. Such columns are filled based on previous fields values by applying `expr`. If `expr` is not present will repeate previous value. Omitted list will result in including all allowed columns.
 
 Example of a query without `WITH FILL`:
 
@@ -483,4 +485,62 @@ Result:
 └────────────┴────────────┴──────────┘
 ```
 
+Example of a query without `INTERPOLATE`:
+
+``` sql
+SELECT n, source, inter FROM (
+   SELECT toFloat32(number % 10) AS n, 'original' AS source, number as inter
+   FROM numbers(10) WHERE number % 3 = 1
+) ORDER BY n WITH FILL FROM 0 TO 5.51 STEP 0.5;
+```
+
+Result:
+
+``` text
+┌───n─┬─source───┬─inter─┐
+│   0 │          │     0 │
+│ 0.5 │          │     0 │
+│   1 │ original │     1 │
+│ 1.5 │          │     0 │
+│   2 │          │     0 │
+│ 2.5 │          │     0 │
+│   3 │          │     0 │
+│ 3.5 │          │     0 │
+│   4 │ original │     4 │
+│ 4.5 │          │     0 │
+│   5 │          │     0 │
+│ 5.5 │          │     0 │
+│   7 │ original │     7 │
+└─────┴──────────┴───────┘
+```
+
+Same query after applying `INTERPOLATE`:
+
+``` sql
+SELECT n, source, inter FROM (
+   SELECT toFloat32(number % 10) AS n, 'original' AS source, number as inter
+   FROM numbers(10) WHERE number % 3 = 1
+) ORDER BY n WITH FILL FROM 0 TO 5.51 STEP 0.5 INTERPOLATE (inter AS inter + 1);
+```
+
+Result:
+
+``` text
+┌───n─┬─source───┬─inter─┐
+│   0 │          │     0 │
+│ 0.5 │          │     0 │
+│   1 │ original │     1 │
+│ 1.5 │          │     2 │
+│   2 │          │     3 │
+│ 2.5 │          │     4 │
+│   3 │          │     5 │
+│ 3.5 │          │     6 │
+│   4 │ original │     4 │
+│ 4.5 │          │     5 │
+│   5 │          │     6 │
+│ 5.5 │          │     7 │
+│   7 │ original │     7 │
+└─────┴──────────┴───────┘
+```
+
 [Original article](https://clickhouse.com/docs/en/sql-reference/statements/select/order-by/) <!--hide-->

docs/ja/introduction/adopters.md

@@ -27,7 +27,7 @@ toc_title: "\u30A2\u30C0\u30D7\u30BF\u30FC"
 | <a href="http://cisco.com/" class="favicon">Cisco</a>                                          | ネットワーク                 | トラフィック分析     | —                                                 | —                                                                        | [ライトニングトーク2019](https://youtu.be/-hI1vDR2oPY?t=5057)                                                                                                                                                     |
 | <a href="https://www.citadelsecurities.com/" class="favicon">Citadel Securities</a>            | 金融                         | —                    | —                                                 | —                                                                        | [2019年の貢献](https://github.com/ClickHouse/ClickHouse/pull/4774)                                                                                                                                                |
 | <a href="https://city-mobil.ru" class="favicon">シティモービル</a>                             | タクシー                     | 分析                 | —                                                 | —                                                                        | [ロシア語でのブログ投稿,月2020](https://habr.com/en/company/citymobil/blog/490660/)                                                                                                                               |
-| <a href="https://contentsquare.com" class="favicon">ContentSquare</a>                          | ウェブ分析                   | 主な製品             | —                                                 | —                                                                        | [フランス語でのブログ投稿,November2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/)                     |
+| <a href="https://contentsquare.com" class="favicon">Contentsquare</a>                          | ウェブ分析                   | 主な製品             | —                                                 | —                                                                        | [フランス語でのブログ投稿,November2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/)                     |
 | <a href="https://cloudflare.com" class="favicon">Cloudflare</a>                                | CDN                          | トラフィック分析     | 36台のサーバー                                    | —                                                                        | [ブログ投稿,月2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [ブログ投稿,月2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) |
 | <a href="https://coru.net/" class="favicon">コルネット</a>                                     | 分析                         | 主な製品             | —                                                 | —                                                                        | [2019年英語スライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf)                                                                                           |
 | <a href="https://www.creditx.com" class="favicon">CraiditX 氪信</a>                            | ファイナンスAI               | 分析                 | —                                                 | —                                                                        | [2019年のスライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx)                                                                                                          |

docs/ru/sql-reference/statements/select/index.md

@@ -20,7 +20,7 @@ SELECT [DISTINCT [ON (column1, column2, ...)]] expr_list
 [WHERE expr]
 [GROUP BY expr_list] [WITH ROLLUP|WITH CUBE] [WITH TOTALS]
 [HAVING expr]
-[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr]
+[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr] [INTERPOLATE [(expr_list)]]
 [LIMIT [offset_value, ]n BY columns]
 [LIMIT [n, ]m] [WITH TIES]
 [SETTINGS ...]

docs/ru/sql-reference/statements/select/order-by.md

@@ -280,6 +280,7 @@ SELECT * FROM collate_test ORDER BY s ASC COLLATE 'en';
 
 ```sql
 ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr]
+[INTERPOLATE [(col [AS expr], ... colN [AS exprN])]]
 ```
 
 `WITH FILL` может быть применен к полям с числовыми (все разновидности float, int, decimal) или временными (все разновидности Date, DateTime) типами. В случае применения к полям типа `String` недостающие значения заполняются пустой строкой.
@@ -289,6 +290,8 @@ ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_
 
 Когда `STEP const_numeric_expr` не указан, тогда используется `1.0` для числовых типов, `1 день` для типа Date и `1 секунда` для типа DateTime.
 
+`INTERPOLATE` может быть применен к колонкам, не участвующим в `ORDER BY WITH FILL`. Такие колонки заполняются значениями, вычисляемыми применением `expr` к предыдущему значению. Если `expr` опущен, то колонка заполняется предыдущим значением. Если список колонок не указан, то включаются все разрешенные колонки.
+
 Пример запроса без использования `WITH FILL`:
 ```sql
 SELECT n, source FROM (
@@ -395,3 +398,58 @@ ORDER BY
 │ 1970-03-12 │ 1970-01-08 │ original │
 └────────────┴────────────┴──────────┘
 ```
+
+Пример запроса без `INTERPOLATE`:
+
+``` sql
+SELECT n, source, inter FROM (
+   SELECT toFloat32(number % 10) AS n, 'original' AS source, number as inter
+   FROM numbers(10) WHERE number % 3 = 1
+) ORDER BY n WITH FILL FROM 0 TO 5.51 STEP 0.5;
+```
+
+Результат:
+``` text
+┌───n─┬─source───┬─inter─┐
+│   0 │          │     0 │
+│ 0.5 │          │     0 │
+│   1 │ original │     1 │
+│ 1.5 │          │     0 │
+│   2 │          │     0 │
+│ 2.5 │          │     0 │
+│   3 │          │     0 │
+│ 3.5 │          │     0 │
+│   4 │ original │     4 │
+│ 4.5 │          │     0 │
+│   5 │          │     0 │
+│ 5.5 │          │     0 │
+│   7 │ original │     7 │
+└─────┴──────────┴───────┘
+```
+
+Тот же запрос с `INTERPOLATE`:
+
+``` sql
+SELECT n, source, inter FROM (
+   SELECT toFloat32(number % 10) AS n, 'original' AS source, number as inter
+   FROM numbers(10) WHERE number % 3 = 1
+) ORDER BY n WITH FILL FROM 0 TO 5.51 STEP 0.5 INTERPOLATE (inter AS inter + 1);
+```
+
+Результат:
+``` text
+┌───n─┬─source───┬─inter─┐
+│   0 │          │     0 │
+│ 0.5 │          │     0 │
+│   1 │ original │     1 │
+│ 1.5 │          │     2 │
+│   2 │          │     3 │
+│ 2.5 │          │     4 │
+│   3 │          │     5 │
+│ 3.5 │          │     6 │
+│   4 │ original │     4 │
+│ 4.5 │          │     5 │
+│   5 │          │     6 │
+│ 5.5 │          │     7 │
+│   7 │ original │     7 │
+└─────┴──────────┴───────┘

docs/zh/operations/configuration-files.md

@@ -3,7 +3,7 @@
 ClickHouse支持多配置文件管理。主配置文件是`/etc/clickhouse-server/config.xml`。其余文件须在目录`/etc/clickhouse-server/config.d`。
 
 !!! 注意：
-    所有配置文件必须是XML格式。此外，配置文件须有相同的跟元素，通常是`<clickhouse>`。
+    所有配置文件必须是XML格式。此外，配置文件须有相同的根元素，通常是`<clickhouse>`。
 
 主配置文件中的一些配置可以通过`replace`或`remove`属性被配置文件覆盖。
 

programs/client/Client.cpp

@@ -163,10 +163,24 @@ void Client::initialize(Poco::Util::Application & self)
 
     configReadClient(config(), home_path);
 
+    /** getenv is thread-safe in Linux glibc and in all sane libc implementations.
+      * But the standard does not guarantee that subsequent calls will not rewrite the value by returned pointer.
+      *
+      * man getenv:
+      *
+      * As typically implemented, getenv() returns a pointer to a string within the environment list.
+      * The caller must take care not to modify this string, since that would change the environment of
+      * the process.
+      *
+      * The implementation of getenv() is not required to be reentrant. The string pointed to by the return value of getenv()
+      * may be statically allocated, and can be modified by a subsequent call to getenv(), putenv(3), setenv(3), or unsetenv(3).
+      */
+
     const char * env_user = getenv("CLICKHOUSE_USER");
-    const char * env_password = getenv("CLICKHOUSE_PASSWORD");
     if (env_user)
         config().setString("user", env_user);
+
+    const char * env_password = getenv("CLICKHOUSE_PASSWORD");
     if (env_password)
         config().setString("password", env_password);
 
@@ -1005,6 +1019,7 @@ void Client::processConfig()
             global_context->setCurrentQueryId(query_id);
     }
     print_stack_trace = config().getBool("stacktrace", false);
+    logging_initialized = true;
 
     if (config().has("multiquery"))
         is_multiquery = true;

programs/format/Format.cpp

@@ -54,6 +54,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
         ("multiquery,n", "allow multiple queries in the same file")
         ("obfuscate", "obfuscate instead of formatting")
         ("backslash", "add a backslash at the end of each line of the formatted query")
+        ("allow_settings_after_format_in_insert", "Allow SETTINGS after FORMAT, but note, that this is not always safe")
         ("seed", po::value<std::string>(), "seed (arbitrary string) that determines the result of obfuscation")
     ;
 
@@ -83,6 +84,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
         bool multiple = options.count("multiquery");
         bool obfuscate = options.count("obfuscate");
         bool backslash = options.count("backslash");
+        bool allow_settings_after_format_in_insert = options.count("allow_settings_after_format_in_insert");
 
         if (quiet && (hilite || oneline || obfuscate))
         {
@@ -154,7 +156,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
             const char * pos = query.data();
             const char * end = pos + query.size();
 
-            ParserQuery parser(end);
+            ParserQuery parser(end, allow_settings_after_format_in_insert);
             do
             {
                 ASTPtr res = parseQueryAndMovePosition(

programs/local/LocalServer.cpp

@@ -434,6 +434,14 @@ catch (...)
     return getCurrentExceptionCode();
 }
 
+void LocalServer::updateLoggerLevel(const String & logs_level)
+{
+    if (!logging_initialized)
+        return;
+
+    config().setString("logger.level", logs_level);
+    updateLevels(config(), logger());
+}
 
 void LocalServer::processConfig()
 {
@@ -460,30 +468,31 @@ void LocalServer::processConfig()
     auto logging = (config().has("logger.console")
                     || config().has("logger.level")
                     || config().has("log-level")
+                    || config().has("send_logs_level")
                     || config().has("logger.log"));
 
-    auto file_logging = config().has("server_logs_file");
-    if (is_interactive && logging && !file_logging)
-        throw Exception("For interactive mode logging is allowed only with --server_logs_file option",
-                        ErrorCodes::BAD_ARGUMENTS);
+    auto level = config().getString("log-level", "trace");
 
-    if (file_logging)
+    if (config().has("server_logs_file"))
     {
-        auto level = Poco::Logger::parseLevel(config().getString("log-level", "trace"));
-        Poco::Logger::root().setLevel(level);
+        auto poco_logs_level = Poco::Logger::parseLevel(level);
+        Poco::Logger::root().setLevel(poco_logs_level);
         Poco::Logger::root().setChannel(Poco::AutoPtr<Poco::SimpleFileChannel>(new Poco::SimpleFileChannel(server_logs_file)));
+        logging_initialized = true;
     }
-    else if (logging)
+    else if (logging || is_interactive)
     {
-        // force enable logging
         config().setString("logger", "logger");
-        // sensitive data rules are not used here
+        auto log_level_default = is_interactive && !logging ? "none" : level;
+        config().setString("logger.level", config().getString("log-level", config().getString("send_logs_level", log_level_default)));
         buildLoggers(config(), logger(), "clickhouse-local");
+        logging_initialized = true;
     }
     else
     {
         Poco::Logger::root().setLevel("none");
         Poco::Logger::root().setChannel(Poco::AutoPtr<Poco::NullChannel>(new Poco::NullChannel()));
+        logging_initialized = false;
     }
 
     shared_context = Context::createShared();
@@ -713,6 +722,8 @@ void LocalServer::processOptions(const OptionsDescription &, const CommandLineOp
         config().setString("logger.log", options["logger.log"].as<std::string>());
     if (options.count("logger.level"))
         config().setString("logger.level", options["logger.level"].as<std::string>());
+    if (options.count("send_logs_level"))
+        config().setString("send_logs_level", options["send_logs_level"].as<std::string>());
 }
 
 }

programs/local/LocalServer.h

@@ -46,6 +46,8 @@ protected:
 
     void processConfig() override;
 
+    void updateLoggerLevel(const String & logs_level) override;
+
 private:
     /** Composes CREATE subquery based on passed arguments (--structure --file --table and --input-format)
       * This query will be executed first, before queries passed through --query argument

programs/server/Server.cpp

@@ -1500,7 +1500,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
     else
     {
         /// Initialize a watcher periodically updating DNS cache
-        dns_cache_updater = std::make_unique<DNSCacheUpdater>(global_context, config().getInt("dns_cache_update_period", 15));
+        dns_cache_updater = std::make_unique<DNSCacheUpdater>(
+            global_context, config().getInt("dns_cache_update_period", 15), config().getUInt("dns_max_consecutive_failures", 5));
     }
 
 #if defined(OS_LINUX)
@@ -1635,6 +1636,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
                 server.start();
                 LOG_INFO(log, "Listening for {}", server.getDescription());
             }
+
+            global_context->setServerCompletelyStarted();
             LOG_INFO(log, "Ready for connections.");
         }
 

programs/server/config.xml

@@ -1042,6 +1042,15 @@
         <flush_interval_milliseconds>7500</flush_interval_milliseconds>
     </session_log> -->
 
+    <!-- Profiling on Processors level. -->
+    <processors_profile_log>
+        <database>system</database>
+        <table>processors_profile_log</table>
+
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </processors_profile_log>
+
     <!-- <top_level_domains_path>/var/lib/clickhouse/top_level_domains/</top_level_domains_path> -->
     <!-- Custom TLD lists.
          Format: <name>/path/to/file</name>

src/Access/Common/AccessType.h

@@ -102,6 +102,7 @@ enum class AccessType
     \
     M(KILL_QUERY, "", GLOBAL, ALL) /* allows to kill a query started by another user
                                       (anyone can kill his own queries) */\
+    M(KILL_TRANSACTION, "", GLOBAL, ALL) \
     \
     M(MOVE_PARTITION_BETWEEN_SHARDS, "", GLOBAL, ALL) /* required to be able to move a part/partition to a table
                                                          identified by its ZooKeeper path */\

src/Backups/ArchiveBackup.cpp

@@ -42,6 +42,14 @@ void ArchiveBackup::openImpl(OpenMode open_mode_)
     /// mutex is already locked
     if (open_mode_ == OpenMode::WRITE)
     {
+        /// Create a directory to contain the archive.
+        auto dir_path = fs::path(path).parent_path();
+        if (disk)
+            disk->createDirectories(dir_path);
+        else
+            std::filesystem::create_directories(dir_path);
+
+        /// Start writing the archive.
         if (disk)
             writer = createArchiveWriter(path, disk->writeFile(path));
         else
@@ -65,7 +73,7 @@ void ArchiveBackup::openImpl(OpenMode open_mode_)
     }
 }
 
-void ArchiveBackup::closeImpl(bool writing_finalized_)
+void ArchiveBackup::closeImpl(const Strings &, bool writing_finalized_)
 {
     /// mutex is already locked
     if (writer && writer->isWritingFile())

src/Backups/ArchiveBackup.h

@@ -35,7 +35,7 @@ public:
 private:
     bool backupExists() const override;
     void openImpl(OpenMode open_mode_) override;
-    void closeImpl(bool writing_finalized_) override;
+    void closeImpl(const Strings & written_files_, bool writing_finalized_) override;
     bool supportsWritingInMultipleThreads() const override { return false; }
     std::unique_ptr<ReadBuffer> readFileImpl(const String & file_name) const override;
     std::unique_ptr<WriteBuffer> writeFileImpl(const String & file_name) override;

src/Backups/BackupImpl.cpp

@@ -107,6 +107,7 @@ void BackupImpl::open(OpenMode open_mode_)
         timestamp = std::time(nullptr);
         uuid = UUIDHelpers::generateV4();
         writing_finalized = false;
+        written_files.clear();
     }
 
     if (open_mode_ == OpenMode::READ)
@@ -145,7 +146,7 @@ void BackupImpl::close()
     if (open_mode == OpenMode::NONE)
         return;
 
-    closeImpl(writing_finalized);
+    closeImpl(written_files, writing_finalized);
 
     uuid = UUIDHelpers::Nil;
     timestamp = 0;
@@ -202,9 +203,12 @@ void BackupImpl::writeBackupMetadata()
             config->setString(prefix + "checksum", getHexUIntLowercase(info.checksum));
             if (info.base_size)
             {
-                config->setUInt(prefix + "base_size", info.base_size);
-                if (info.base_checksum != info.checksum)
+                config->setBool(prefix + "use_base", true);
+                if (info.base_size != info.size)
+                {
+                    config->setUInt(prefix + "base_size", info.base_size);
                     config->setString(prefix + "base_checksum", getHexUIntLowercase(info.base_checksum));
+                }
             }
         }
         ++index;
@@ -213,6 +217,7 @@ void BackupImpl::writeBackupMetadata()
     std::ostringstream stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
     config->save(stream);
     String str = stream.str();
+    written_files.push_back(".backup");
     auto out = writeFileImpl(".backup");
     out->write(str.data(), str.size());
 }
@@ -253,13 +258,14 @@ void BackupImpl::readBackupMetadata()
             if (info.size)
             {
                 info.checksum = unhexChecksum(config->getString(prefix + "checksum"));
-                info.base_size = config->getUInt(prefix + "base_size", 0);
+                bool use_base = config->getBool(prefix + "use_base", false);
+                info.base_size = config->getUInt(prefix + "base_size", use_base ? info.size : 0);
                 if (info.base_size)
                 {
-                    if (config->has(prefix + "base_checksum"))
-                        info.base_checksum = unhexChecksum(config->getString(prefix + "base_checksum"));
-                    else
+                    if (info.base_size == info.size)
                         info.base_checksum = info.checksum;
+                    else
+                        info.base_checksum = unhexChecksum(config->getString(prefix + "base_checksum"));
                 }
             }
             file_infos.emplace(name, info);
@@ -345,11 +351,6 @@ BackupEntryPtr BackupImpl::readFile(const String & file_name) const
         return std::make_unique<BackupEntryFromMemory>(nullptr, 0, UInt128{0, 0});
     }
 
-    auto read_callback = [backup = std::static_pointer_cast<const BackupImpl>(shared_from_this()), file_name]()
-    {
-        return backup->readFileImpl(file_name);
-    };
-
     if (!info.base_size)
     {
         /// Data goes completely from this backup, the base backup isn't used.
@@ -526,6 +527,7 @@ void BackupImpl::writeFile(const String & file_name, BackupEntryPtr entry)
     }
 
     /// Copy the entry's data after `copy_pos`.
+    written_files.push_back(file_name);
     auto out = writeFileImpl(file_name);
     copyData(*read_buffer, *out);
 

src/Backups/BackupImpl.h

@@ -47,7 +47,7 @@ protected:
     virtual void openImpl(OpenMode open_mode_) = 0;
     OpenMode getOpenModeNoLock() const { return open_mode; }
 
-    virtual void closeImpl(bool writing_finalized_) = 0;
+    virtual void closeImpl(const Strings & written_files_, bool writing_finalized_) = 0;
 
     /// Read a file from the backup.
     /// Low level: the function doesn't check base backup or checksums.
@@ -86,6 +86,7 @@ private:
     std::optional<UUID> base_backup_uuid;
     std::map<String, FileInfo> file_infos; /// Should be ordered alphabetically, see listFiles().
     std::unordered_map<UInt128, String> file_checksums;
+    Strings written_files;
     bool writing_finalized = false;
 };
 

src/Backups/BackupInfo.cpp

@@ -1,6 +1,7 @@
 #include <Backups/BackupInfo.h>
 #include <Parsers/ASTExpressionList.h>
 #include <Parsers/ASTFunction.h>
+#include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ExpressionElementParsers.h>
 #include <Parsers/formatAST.h>
@@ -23,7 +24,11 @@ String BackupInfo::toString() const
     auto list = std::make_shared<ASTExpressionList>();
     func->arguments = list;
     func->children.push_back(list);
-    list->children.reserve(args.size());
+    list->children.reserve(args.size() + !id_arg.empty());
+
+    if (!id_arg.empty())
+        list->children.push_back(std::make_shared<ASTIdentifier>(id_arg));
+
     for (const auto & arg : args)
         list->children.push_back(std::make_shared<ASTLiteral>(arg));
 
@@ -53,9 +58,22 @@ BackupInfo BackupInfo::fromAST(const IAST & ast)
         const auto * list = func->arguments->as<const ASTExpressionList>();
         if (!list)
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected list, got {}", serializeAST(*func->arguments));
-        res.args.reserve(list->children.size());
-        for (const auto & elem : list->children)
+
+        size_t index = 0;
+        if (!list->children.empty())
         {
+            const auto * id = list->children[0]->as<const ASTIdentifier>();
+            if (id)
+            {
+                res.id_arg = id->name();
+                ++index;
+            }
+        }
+
+        res.args.reserve(list->children.size() - index);
+        for (; index < list->children.size(); ++index)
+        {
+            const auto & elem = list->children[index];
             const auto * lit = elem->as<const ASTLiteral>();
             if (!lit)
                 throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected literal, got {}", serializeAST(*elem));

src/Backups/BackupInfo.h

@@ -11,6 +11,7 @@ class IAST;
 struct BackupInfo
 {
     String backup_engine_name;
+    String id_arg;
     std::vector<Field> args;
 
     String toString() const;

src/Backups/DirectoryBackup.cpp

@@ -1,16 +1,9 @@
 #include <Backups/DirectoryBackup.h>
-#include <Common/quoteString.h>
-#include <Disks/IDisk.h>
 #include <Disks/DiskLocal.h>
 
 
 namespace DB
 {
-namespace ErrorCodes
-{
-    extern const int BAD_ARGUMENTS;
-}
-
 
 DirectoryBackup::DirectoryBackup(
     const String & backup_name_,
@@ -19,23 +12,16 @@ DirectoryBackup::DirectoryBackup(
     const ContextPtr & context_,
     const std::optional<BackupInfo> & base_backup_info_)
     : BackupImpl(backup_name_, context_, base_backup_info_)
-    , disk(disk_), path(path_)
+    , disk(disk_)
 {
-    /// Path to backup must end with '/'
-    if (!path.ends_with("/"))
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Backup {}: Path to backup must end with '/', but {} doesn't.", getName(), quoteString(path));
-    dir_path = fs::path(path).parent_path(); /// get path without terminating slash
+    /// Remove terminating slash.
+    path = (std::filesystem::path(path_) / "").parent_path();
 
     /// If `disk` is not specified, we create an internal instance of `DiskLocal` here.
     if (!disk)
     {
-        auto fspath = fs::path{dir_path};
-        if (!fspath.has_filename())
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Backup {}: Path to a backup must be a directory path.", getName(), quoteString(path));
-        path = fspath.filename() / "";
-        dir_path = fs::path(path).parent_path(); /// get path without terminating slash
-        String disk_path = fspath.remove_filename();
-        disk = std::make_shared<DiskLocal>(disk_path, disk_path, 0);
+        disk = std::make_shared<DiskLocal>(path, path, 0);
+        path = ".";
     }
 }
 
@@ -47,34 +33,38 @@ DirectoryBackup::~DirectoryBackup()
 
 bool DirectoryBackup::backupExists() const
 {
-    return disk->isDirectory(dir_path);
+    return disk->isDirectory(path);
 }
 
 void DirectoryBackup::openImpl(OpenMode open_mode_)
 {
     if (open_mode_ == OpenMode::WRITE)
-        disk->createDirectories(dir_path);
+        disk->createDirectories(path);
 }
 
-void DirectoryBackup::closeImpl(bool writing_finalized_)
+void DirectoryBackup::closeImpl(const Strings & written_files_, bool writing_finalized_)
 {
-    if ((getOpenModeNoLock() == OpenMode::WRITE) && !writing_finalized_ && disk->isDirectory(dir_path))
+    if ((getOpenModeNoLock() == OpenMode::WRITE) && !writing_finalized_ && !written_files_.empty())
     {
         /// Creating of the backup wasn't finished correctly,
         /// so the backup cannot be used and it's better to remove its files.
-        disk->removeRecursive(dir_path);
+        const auto & files_to_delete = written_files_;
+        for (const String & file_name : files_to_delete)
+            disk->removeFileIfExists(path / file_name);
+        if (disk->isDirectory(path) && disk->isDirectoryEmpty(path))
+            disk->removeDirectory(path);
     }
 }
 
 std::unique_ptr<ReadBuffer> DirectoryBackup::readFileImpl(const String & file_name) const
 {
-    String file_path = path + file_name;
+    auto file_path = path / file_name;
     return disk->readFile(file_path);
 }
 
 std::unique_ptr<WriteBuffer> DirectoryBackup::writeFileImpl(const String & file_name)
 {
-    String file_path = path + file_name;
+    auto file_path = path / file_name;
     disk->createDirectories(fs::path(file_path).parent_path());
     return disk->writeFile(file_path);
 }

src/Backups/DirectoryBackup.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Backups/BackupImpl.h>
+#include <filesystem>
 
 
 namespace DB
@@ -25,13 +26,12 @@ public:
 private:
     bool backupExists() const override;
     void openImpl(OpenMode open_mode_) override;
-    void closeImpl(bool writing_finalized_) override;
+    void closeImpl(const Strings & written_files_, bool writing_finalized_) override;
     std::unique_ptr<ReadBuffer> readFileImpl(const String & file_name) const override;
     std::unique_ptr<WriteBuffer> writeFileImpl(const String & file_name) override;
 
     DiskPtr disk;
-    String path;
-    String dir_path; /// `path` without terminating slash
+    std::filesystem::path path;
 };
 
 }

src/Backups/registerBackupEnginesFileAndDisk.cpp

@@ -2,6 +2,7 @@
 #include <Backups/DirectoryBackup.h>
 #include <Backups/ArchiveBackup.h>
 #include <Common/quoteString.h>
+#include <Disks/IDisk.h>
 #include <IO/Archives/hasRegisteredArchiveFileExtension.h>
 #include <Interpreters/Context.h>
 #include <Poco/Util/AbstractConfiguration.h>
@@ -13,8 +14,9 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int BAD_ARGUMENTS;
-    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
     extern const int INVALID_CONFIG_PARAMETER;
+    extern const int LOGICAL_ERROR;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
 }
 
 
@@ -22,83 +24,71 @@ namespace
 {
     namespace fs = std::filesystem;
 
-    [[noreturn]] void throwDiskIsAllowed(const String & disk_name)
-    {
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Disk {} is not allowed for backups", disk_name);
-    }
-
-    [[noreturn]] void throwPathNotAllowed(const fs::path & path)
-    {
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path {} is not allowed for backups", quoteString(String{path}));
-    }
-
-    void checkAllowedPathInConfigIsValid(const String & key, const fs::path & value)
-    {
-        if (value.empty() || value.is_relative())
-            throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Configuration parameter {} has a wrong value {}", key, String{value});
-    }
-
-    /// Checks that a disk name and a path specified as parameters of Disk() are valid.
-    void checkDiskNameAndPath(const String & disk_name, fs::path & path, const Poco::Util::AbstractConfiguration & config)
+    /// Checks that a disk name specified as parameters of Disk() is valid.
+    void checkDiskName(const String & disk_name, const Poco::Util::AbstractConfiguration & config)
     {
         String key = "backups.allowed_disk";
-        bool disk_name_found = false;
+        if (!config.has(key))
+            throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "The \"backups.allowed_disk\" configuration parameter is not set, cannot use Disk() backup engine");
+
         size_t counter = 0;
-        while (config.has(key))
+        while (config.getString(key) != disk_name)
         {
-            if (config.getString(key) == disk_name)
-            {
-                disk_name_found = true;
-                break;
-            }
             key = "backups.allowed_disk[" + std::to_string(++counter) + "]";
+            if (!config.has(key))
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Disk {} is not allowed for backups, see the \"backups.allowed_disk\" configuration parameter", disk_name);
         }
-
-        if (!disk_name_found)
-            throwDiskIsAllowed(disk_name);
-
-        path = path.lexically_normal();
-        if (!path.is_relative() || path.empty() || (*path.begin() == ".."))
-            throwPathNotAllowed(path);
     }
 
-    /// Checks that a path specified as a parameter of File() is valid.
-    void checkPath(fs::path & path, const Poco::Util::AbstractConfiguration & config)
+    /// Checks that a path specified as parameters of Disk() is valid.
+    void checkPath(const String & disk_name, const DiskPtr & disk, fs::path & path)
     {
-        String key = "backups.allowed_path";
+        path = path.lexically_normal();
+        if (!path.is_relative() && (disk->getType() == DiskType::Local))
+            path = path.lexically_proximate(disk->getPath());
 
+        bool path_ok = path.empty() || (path.is_relative() && (*path.begin() != ".."));
+        if (!path_ok)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path {} to backup must be inside the specified disk {}", quoteString(path.c_str()), disk_name);
+    }
+
+    /// Checks that a path specified as parameters of File() is valid.
+    void checkPath(fs::path & path, const Poco::Util::AbstractConfiguration & config, const fs::path & data_dir)
+    {
         path = path.lexically_normal();
         if (path.empty())
-            throwPathNotAllowed(path);
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path to backup must not be empty");
+
+        String key = "backups.allowed_path";
+        if (!config.has(key))
+            throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER,
+                            "The \"backups.allowed_path\" configuration parameter is not set, cannot use File() backup engine");
 
         if (path.is_relative())
         {
-            if (*path.begin() == "..")
-                throwPathNotAllowed(path);
+            auto first_allowed_path = fs::path(config.getString(key));
+            if (first_allowed_path.is_relative())
+                first_allowed_path = data_dir / first_allowed_path;
 
-            auto base = fs::path(config.getString(key, ""));
-            checkAllowedPathInConfigIsValid(key, base);
-            path = base / path;
-            return;
+            path = first_allowed_path / path;
         }
 
-        bool path_found_in_config = false;
         size_t counter = 0;
-        while (config.has(key))
+        while (true)
         {
-            auto base = fs::path(config.getString(key));
-            checkAllowedPathInConfigIsValid(key, base);
-            auto rel = path.lexically_relative(base);
-            if (!rel.empty() && (*rel.begin() != ".."))
-            {
-                path_found_in_config = true;
+            auto allowed_path = fs::path(config.getString(key));
+            if (allowed_path.is_relative())
+                allowed_path = data_dir / allowed_path;
+            auto rel = path.lexically_proximate(allowed_path);
+            bool path_ok = rel.empty() || (rel.is_relative() && (*rel.begin() != ".."));
+            if (path_ok)
                 break;
-            }
             key = "backups.allowed_path[" + std::to_string(++counter) + "]";
+            if (!config.has(key))
+                throw Exception(ErrorCodes::BAD_ARGUMENTS,
+                                "Path {} is not allowed for backups, see the \"backups.allowed_path\" configuration parameter",
+                                quoteString(path.c_str()));
         }
-
-        if (!path_found_in_config)
-            throwPathNotAllowed(path);
     }
 }
 
@@ -109,6 +99,15 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
     {
         String backup_name = params.backup_info.toString();
         const String & engine_name = params.backup_info.backup_engine_name;
+
+        if (!params.backup_info.id_arg.empty())
+        {
+            throw Exception(
+                ErrorCodes::BAD_ARGUMENTS,
+                "Backup engine '{}' requires first argument to be a string",
+                engine_name);
+        }
+
         const auto & args = params.backup_info.args;
 
         DiskPtr disk;
@@ -123,7 +122,9 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
             }
 
             path = args[0].safeGet<String>();
-            checkPath(path, params.context->getConfigRef());
+            const auto & config = params.context->getConfigRef();
+            const auto & data_dir = params.context->getPath();
+            checkPath(path, config, data_dir);
         }
         else if (engine_name == "Disk")
         {
@@ -135,30 +136,28 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
             }
 
             String disk_name = args[0].safeGet<String>();
+            const auto & config = params.context->getConfigRef();
+            checkDiskName(disk_name, config);
             path = args[1].safeGet<String>();
-            checkDiskNameAndPath(disk_name, path, params.context->getConfigRef());
             disk = params.context->getDisk(disk_name);
+            checkPath(disk_name, disk, path);
         }
+        else
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected backup engine '{}'", engine_name);
 
-        std::unique_ptr<IBackup> backup;
-
-        if (!path.has_filename() && !path.empty())
-        {
-            if (!params.password.empty())
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Password is not applicable, backup cannot be encrypted");
-            backup = std::make_unique<DirectoryBackup>(backup_name, disk, path, params.context, params.base_backup_info);
-        }
-        else if (hasRegisteredArchiveFileExtension(path))
+        if (hasRegisteredArchiveFileExtension(path))
         {
             auto archive_backup = std::make_unique<ArchiveBackup>(backup_name, disk, path, params.context, params.base_backup_info);
             archive_backup->setCompression(params.compression_method, params.compression_level);
             archive_backup->setPassword(params.password);
-            backup = std::move(archive_backup);
+            return archive_backup;
         }
         else
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path to backup must be either a directory or a path to an archive");
-
-        return backup;
+        {
+            if (!params.password.empty())
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Password is not applicable, backup cannot be encrypted");
+            return std::make_unique<DirectoryBackup>(backup_name, disk, path, params.context, params.base_backup_info);
+        }
     };
 
     factory.registerBackupEngine("File", creator_fn);

src/Client/ClientBase.cpp

@@ -240,6 +240,14 @@ void interruptSignalHandler(int signum)
 }
 
 
+/// To cancel the query on local format error.
+class LocalFormatError : public DB::Exception
+{
+public:
+    using Exception::Exception;
+};
+
+
 ClientBase::~ClientBase() = default;
 ClientBase::ClientBase() = default;
 
@@ -268,7 +276,7 @@ void ClientBase::setupSignalHandler()
 
 ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, bool allow_multi_statements) const
 {
-    ParserQuery parser(end);
+    ParserQuery parser(end, global_context->getSettings().allow_settings_after_format_in_insert);
     ASTPtr res;
 
     const auto & settings = global_context->getSettingsRef();
@@ -442,6 +450,7 @@ void ClientBase::onProfileInfo(const ProfileInfo & profile_info)
 
 
 void ClientBase::initBlockOutputStream(const Block & block, ASTPtr parsed_query)
+try
 {
     if (!output_format)
     {
@@ -530,6 +539,10 @@ void ClientBase::initBlockOutputStream(const Block & block, ASTPtr parsed_query)
         output_format->setAutoFlush();
     }
 }
+catch (...)
+{
+    throw LocalFormatError(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode());
+}
 
 
 void ClientBase::initLogsOutputStream()
@@ -721,6 +734,9 @@ void ClientBase::receiveResult(ASTPtr parsed_query)
         = std::max(min_poll_interval, std::min<size_t>(receive_timeout.totalMicroseconds(), default_poll_interval));
 
     bool break_on_timeout = connection->getConnectionType() != IServerConnection::Type::LOCAL;
+
+    std::exception_ptr local_format_error;
+
     while (true)
     {
         Stopwatch receive_watch(CLOCK_MONOTONIC_COARSE);
@@ -769,10 +785,21 @@ void ClientBase::receiveResult(ASTPtr parsed_query)
                 break;
         }
 
-        if (!receiveAndProcessPacket(parsed_query, cancelled))
-            break;
+        try
+        {
+            if (!receiveAndProcessPacket(parsed_query, cancelled))
+                break;
+        }
+        catch (const LocalFormatError &)
+        {
+            local_format_error = std::current_exception();
+            connection->sendCancel();
+        }
     }
 
+    if (local_format_error)
+        std::rethrow_exception(local_format_error);
+
     if (cancelled && is_interactive)
         std::cout << "Query was cancelled." << std::endl;
 }
@@ -1102,7 +1129,7 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des
             sendDataFromPipe(
                 storage->read(
                         sample.getNames(),
-                        storage->getStorageSnapshot(metadata),
+                        storage->getStorageSnapshot(metadata, global_context),
                         query_info,
                         global_context,
                         {},
@@ -1298,6 +1325,13 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
         }
     }
 
+    if (const auto * set_query = parsed_query->as<ASTSetQuery>())
+    {
+        const auto * logs_level_field = set_query->changes.tryGet(std::string_view{"send_logs_level"});
+        if (logs_level_field)
+            updateLoggerLevel(logs_level_field->safeGet<String>());
+    }
+
     processed_rows = 0;
     written_first_block = false;
     progress_indication.resetProgress();
@@ -1494,24 +1528,19 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
 
 bool ClientBase::executeMultiQuery(const String & all_queries_text)
 {
-    // It makes sense not to base any control flow on this, so that it is
-    // the same in tests and in normal usage. The only difference is that in
-    // normal mode we ignore the test hints.
-    const bool test_mode = config().has("testmode");
-    if (test_mode)
-    {
-        /// disable logs if expects errors
-        TestHint test_hint(test_mode, all_queries_text);
-        if (test_hint.clientError() || test_hint.serverError())
-            processTextAsSingleQuery("SET send_logs_level = 'fatal'");
-    }
-
     bool echo_query = echo_queries;
 
     /// Test tags are started with "--" so they are interpreted as comments anyway.
     /// But if the echo is enabled we have to remove the test tags from `all_queries_text`
     /// because we don't want test tags to be echoed.
-    size_t test_tags_length = test_mode ? getTestTagsLength(all_queries_text) : 0;
+    {
+        /// disable logs if expects errors
+        TestHint test_hint(all_queries_text);
+        if (test_hint.clientError() || test_hint.serverError())
+            processTextAsSingleQuery("SET send_logs_level = 'fatal'");
+    }
+
+    size_t test_tags_length = getTestTagsLength(all_queries_text);
 
     /// Several queries separated by ';'.
     /// INSERT data is ended by the end of line, not ';'.
@@ -1548,7 +1577,7 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
                 // Try to find test hint for syntax error. We don't know where
                 // the query ends because we failed to parse it, so we consume
                 // the entire line.
-                TestHint hint(test_mode, String(this_query_begin, this_query_end - this_query_begin));
+                TestHint hint(String(this_query_begin, this_query_end - this_query_begin));
                 if (hint.serverError())
                 {
                     // Syntax errors are considered as client errors
@@ -1586,7 +1615,7 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
                 // Look for the hint in the text of query + insert data + trailing
                 // comments, e.g. insert into t format CSV 'a' -- { serverError 123 }.
                 // Use the updated query boundaries we just calculated.
-                TestHint test_hint(test_mode, full_query);
+                TestHint test_hint(full_query);
 
                 // Echo all queries if asked; makes for a more readable reference file.
                 echo_query = test_hint.echoQueries().value_or(echo_query);
@@ -2187,8 +2216,6 @@ void ClientBase::init(int argc, char ** argv)
         ("suggestion_limit", po::value<int>()->default_value(10000),
             "Suggestion limit for how many databases, tables and columns to fetch.")
 
-        ("testmode,T", "enable test hints in comments")
-
         ("format,f", po::value<std::string>(), "default output format")
         ("vertical,E", "vertical output format, same as --format=Vertical or FORMAT Vertical or \\G at end of command")
         ("highlight", po::value<bool>()->default_value(true), "enable or disable basic syntax highlight in interactive command line")
@@ -2294,8 +2321,6 @@ void ClientBase::init(int argc, char ** argv)
         config().setBool("interactive", true);
     if (options.count("pager"))
         config().setString("pager", options["pager"].as<std::string>());
-    if (options.count("testmode"))
-        config().setBool("testmode", true);
 
     if (options.count("log-level"))
         Poco::Logger::root().setLevel(options["log-level"].as<std::string>());

src/Client/ClientBase.h

@@ -95,6 +95,7 @@ protected:
         std::optional<ProgramOptionsDescription> hosts_and_ports_description;
     };
 
+    virtual void updateLoggerLevel(const String &) {}
     virtual void printHelpMessage(const OptionsDescription & options_description) = 0;
     virtual void addOptions(OptionsDescription & options_description) = 0;
     virtual void processOptions(const OptionsDescription & options_description,
@@ -265,6 +266,8 @@ protected:
     bool allow_repeated_settings = false;
 
     bool cancelled = false;
+
+    bool logging_initialized = false;
 };
 
 }

src/Client/TestHint.cpp

@@ -32,12 +32,9 @@ int parseErrorCode(DB::ReadBufferFromString & in)
 namespace DB
 {
 
-TestHint::TestHint(bool enabled_, const String & query_)
+TestHint::TestHint(const String & query_)
     : query(query_)
 {
-    if (!enabled_)
-        return;
-
     // Don't parse error hints in leading comments, because it feels weird.
     // Leading 'echo' hint is OK.
     bool is_leading_hint = true;

src/Client/TestHint.h

@@ -7,7 +7,7 @@
 namespace DB
 {
 
-/// Checks expected server and client error codes in --testmode.
+/// Checks expected server and client error codes.
 ///
 /// The following comment hints are supported:
 ///
@@ -25,12 +25,12 @@ namespace DB
 ///
 /// Examples:
 ///
-/// - echo 'select / -- { clientError 62 }' | clickhouse-client --testmode -nm
+/// - echo 'select / -- { clientError 62 }' | clickhouse-client -nm
 ///
 //    Here the client parses the query but it is incorrect, so it expects
 ///   SYNTAX_ERROR (62).
 ///
-/// - echo 'select foo -- { serverError 47 }' | clickhouse-client --testmode -nm
+/// - echo 'select foo -- { serverError 47 }' | clickhouse-client -nm
 ///
 ///   But here the query is correct, but there is no such column "foo", so it
 ///   is UNKNOWN_IDENTIFIER server error.
@@ -43,7 +43,7 @@ namespace DB
 class TestHint
 {
 public:
-    TestHint(bool enabled_, const String & query_);
+    TestHint(const String & query_);
 
     int serverError() const { return server_error; }
     int clientError() const { return client_error; }

src/Common/CurrentMetrics.cpp

@@ -81,6 +81,14 @@
     M(ActiveSyncDrainedConnections, "Number of active connections drained synchronously.") \
     M(AsynchronousReadWait, "Number of threads waiting for asynchronous read.") \
     M(PendingAsyncInsert, "Number of asynchronous inserts that are waiting for flush.") \
+    M(KafkaConsumers, "Number of active Kafka consumers") \
+    M(KafkaConsumersWithAssignment, "Number of active Kafka consumers which have some partitions assigned.") \
+    M(KafkaProducers, "Number of active Kafka producer created") \
+    M(KafkaLibrdkafkaThreads, "Number of active librdkafka threads") \
+    M(KafkaBackgroundReads, "Number of background reads currently working (populating materialized views from Kafka)") \
+    M(KafkaConsumersInUse, "Number of consumers which are currently used by direct or background reads") \
+    M(KafkaWrites, "Number of currently running inserts to Kafka") \
+    M(KafkaAssignedPartitions, "Number of partitions Kafka tables currently assigned to") \
 
 namespace CurrentMetrics
 {

src/Common/CurrentThread.h

@@ -91,6 +91,7 @@ public:
     struct QueryScope
     {
         explicit QueryScope(ContextMutablePtr query_context);
+        explicit QueryScope(ContextPtr query_context);
         ~QueryScope();
 
         void logPeakMemoryUsage();

src/Common/DNSResolver.cpp

@@ -118,12 +118,15 @@ static DNSResolver::IPAddresses resolveIPAddressImpl(const std::string & host)
     }
     catch (const Poco::Net::DNSException & e)
     {
-        LOG_ERROR(&Poco::Logger::get("DNSResolver"), "Cannot resolve host ({}), error {}: {}.", host, e.code(), e.message());
+        LOG_ERROR(&Poco::Logger::get("DNSResolver"), "Cannot resolve host ({}), error {}: {}.", host, e.code(), e.name());
         addresses.clear();
     }
 
     if (addresses.empty())
+    {
+        ProfileEvents::increment(ProfileEvents::DNSError);
         throw Exception("Not found address of host: " + host, ErrorCodes::DNS_ERROR);
+    }
 
     return addresses;
 }
@@ -142,6 +145,9 @@ static String reverseResolveImpl(const Poco::Net::IPAddress & address)
 
 struct DNSResolver::Impl
 {
+    using HostWithConsecutiveFailures = std::unordered_map<String, UInt32>;
+    using AddressWithConsecutiveFailures = std::unordered_map<Poco::Net::IPAddress, UInt32>;
+
     CachedFn<&resolveIPAddressImpl> cache_host;
     CachedFn<&reverseResolveImpl> cache_address;
 
@@ -152,12 +158,12 @@ struct DNSResolver::Impl
     std::optional<String> host_name;
 
     /// Store hosts, which was asked to resolve from last update of DNS cache.
-    NameSet new_hosts;
-    std::unordered_set<Poco::Net::IPAddress> new_addresses;
+    HostWithConsecutiveFailures new_hosts;
+    AddressWithConsecutiveFailures new_addresses;
 
     /// Store all hosts, which was whenever asked to resolve
-    NameSet known_hosts;
-    std::unordered_set<Poco::Net::IPAddress> known_addresses;
+    HostWithConsecutiveFailures known_hosts;
+    AddressWithConsecutiveFailures known_addresses;
 
     /// If disabled, will not make cache lookups, will resolve addresses manually on each call
     std::atomic<bool> disable_cache{false};
@@ -246,38 +252,68 @@ String DNSResolver::getHostName()
 static const String & cacheElemToString(const String & str) { return str; }
 static String cacheElemToString(const Poco::Net::IPAddress & addr) { return addr.toString(); }
 
-template<typename UpdateF, typename ElemsT>
-bool DNSResolver::updateCacheImpl(UpdateF && update_func, ElemsT && elems, const String & log_msg)
+template <typename UpdateF, typename ElemsT>
+bool DNSResolver::updateCacheImpl(
+    UpdateF && update_func,
+    ElemsT && elems,
+    UInt32 max_consecutive_failures,
+    const String & notfound_log_msg,
+    const String & dropped_log_msg)
 {
     bool updated = false;
     String lost_elems;
-    for (const auto & elem : elems)
+    using iterators = typename std::remove_reference_t<decltype(elems)>::iterator;
+    std::vector<iterators> elements_to_drop;
+    for (auto it = elems.begin(); it != elems.end(); it++)
     {
         try
         {
-            updated |= (this->*update_func)(elem);
+            updated |= (this->*update_func)(it->first);
+            it->second = 0;
         }
-        catch (const Poco::Net::NetException &)
+        catch (const DB::Exception & e)
         {
-            ProfileEvents::increment(ProfileEvents::DNSError);
-
+            if (e.code() != ErrorCodes::DNS_ERROR)
+            {
+                tryLogCurrentException(log, __PRETTY_FUNCTION__);
+                continue;
+            }
             if (!lost_elems.empty())
                 lost_elems += ", ";
-            lost_elems += cacheElemToString(elem);
+            lost_elems += cacheElemToString(it->first);
+            if (max_consecutive_failures)
+            {
+                it->second++;
+                if (it->second >= max_consecutive_failures)
+                    elements_to_drop.emplace_back(it);
+            }
         }
         catch (...)
         {
-            tryLogCurrentException(__PRETTY_FUNCTION__);
+            tryLogCurrentException(log, __PRETTY_FUNCTION__);
         }
     }
 
     if (!lost_elems.empty())
-        LOG_INFO(log, fmt::runtime(log_msg), lost_elems);
+        LOG_INFO(log, fmt::runtime(notfound_log_msg), lost_elems);
+    if (elements_to_drop.size())
+    {
+        updated = true;
+        String deleted_elements;
+        for (auto it : elements_to_drop)
+        {
+            if (!deleted_elements.empty())
+                deleted_elements += ", ";
+            deleted_elements += cacheElemToString(it->first);
+            elems.erase(it);
+        }
+        LOG_INFO(log, fmt::runtime(dropped_log_msg), deleted_elements);
+    }
 
     return updated;
 }
 
-bool DNSResolver::updateCache()
+bool DNSResolver::updateCache(UInt32 max_consecutive_failures)
 {
     LOG_DEBUG(log, "Updating DNS cache");
 
@@ -301,8 +337,14 @@ bool DNSResolver::updateCache()
     /// DROP DNS CACHE will wait on update_mutex (possibly while holding drop_mutex)
     std::lock_guard lock(impl->update_mutex);
 
-    bool hosts_updated = updateCacheImpl(&DNSResolver::updateHost, impl->known_hosts, "Cached hosts not found: {}");
-    updateCacheImpl(&DNSResolver::updateAddress, impl->known_addresses, "Cached addresses not found: {}");
+    bool hosts_updated = updateCacheImpl(
+        &DNSResolver::updateHost, impl->known_hosts, max_consecutive_failures, "Cached hosts not found: {}", "Cached hosts dropped: {}");
+    updateCacheImpl(
+        &DNSResolver::updateAddress,
+        impl->known_addresses,
+        max_consecutive_failures,
+        "Cached addresses not found: {}",
+        "Cached addresses dropped: {}");
 
     LOG_DEBUG(log, "Updated DNS cache");
     return hosts_updated;
@@ -326,13 +368,15 @@ bool DNSResolver::updateAddress(const Poco::Net::IPAddress & address)
 void DNSResolver::addToNewHosts(const String & host)
 {
     std::lock_guard lock(impl->drop_mutex);
-    impl->new_hosts.insert(host);
+    UInt8 consecutive_failures = 0;
+    impl->new_hosts.insert({host, consecutive_failures});
 }
 
 void DNSResolver::addToNewAddresses(const Poco::Net::IPAddress & address)
 {
     std::lock_guard lock(impl->drop_mutex);
-    impl->new_addresses.insert(address);
+    UInt8 consecutive_failures = 0;
+    impl->new_addresses.insert({address, consecutive_failures});
 }
 
 DNSResolver::~DNSResolver() = default;

src/Common/DNSResolver.h

@@ -47,14 +47,20 @@ public:
     void dropCache();
 
     /// Updates all known hosts in cache.
-    /// Returns true if IP of any host has been changed.
-    bool updateCache();
+    /// Returns true if IP of any host has been changed or an element was dropped (too many failures)
+    bool updateCache(UInt32 max_consecutive_failures);
 
     ~DNSResolver();
 
 private:
-    template<typename UpdateF, typename ElemsT>
-    bool updateCacheImpl(UpdateF && update_func, ElemsT && elems, const String & log_msg);
+    template <typename UpdateF, typename ElemsT>
+
+    bool updateCacheImpl(
+        UpdateF && update_func,
+        ElemsT && elems,
+        UInt32 max_consecutive_failures,
+        const String & notfound_log_msg,
+        const String & dropped_log_msg);
 
     DNSResolver();
 

src/Common/ErrorCodes.cpp

@@ -617,6 +617,8 @@
     M(646, CANNOT_BACKUP_DATABASE) \
     M(647, CANNOT_BACKUP_TABLE) \
     M(648, WRONG_DDL_RENAMING_SETTINGS) \
+    M(649, INVALID_TRANSACTION) \
+    M(650, SERIALIZATION_ERROR) \
     \
     M(999, KEEPER_EXCEPTION) \
     M(1000, POCO_EXCEPTION) \

src/Common/FileCache.cpp

@@ -3,6 +3,7 @@
 #include <Common/randomSeed.h>
 #include <Common/SipHash.h>
 #include <Common/hex.h>
+#include <Common/FileCacheSettings.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteBufferFromFile.h>
 #include <IO/ReadSettings.h>
@@ -31,13 +32,11 @@ namespace
 
 IFileCache::IFileCache(
     const String & cache_base_path_,
-    size_t max_size_,
-    size_t max_element_size_,
-    size_t max_file_segment_size_)
+    const FileCacheSettings & cache_settings_)
     : cache_base_path(cache_base_path_)
-    , max_size(max_size_)
-    , max_element_size(max_element_size_)
-    , max_file_segment_size(max_file_segment_size_)
+    , max_size(cache_settings_.max_size)
+    , max_element_size(cache_settings_.max_elements)
+    , max_file_segment_size(cache_settings_.max_file_segment_size)
 {
 }
 
@@ -58,7 +57,7 @@ String IFileCache::getPathInLocalCache(const Key & key)
     return fs::path(cache_base_path) / key_str.substr(0, 3) / key_str;
 }
 
-bool IFileCache::shouldBypassCache()
+bool IFileCache::isReadOnly()
 {
     return !CurrentThread::isInitialized()
         || !CurrentThread::get().getQueryContext()
@@ -71,8 +70,8 @@ void IFileCache::assertInitialized() const
         throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Cache not initialized");
 }
 
-LRUFileCache::LRUFileCache(const String & cache_base_path_, size_t max_size_, size_t max_element_size_, size_t max_file_segment_size_)
-    : IFileCache(cache_base_path_, max_size_, max_element_size_, max_file_segment_size_)
+LRUFileCache::LRUFileCache(const String & cache_base_path_, const FileCacheSettings & cache_settings_)
+    : IFileCache(cache_base_path_, cache_settings_)
     , log(&Poco::Logger::get("LRUFileCache"))
 {
 }
@@ -205,8 +204,8 @@ FileSegments LRUFileCache::getImpl(
     return result;
 }
 
-FileSegments LRUFileCache::splitRangeIntoEmptyCells(
-    const Key & key, size_t offset, size_t size, std::lock_guard<std::mutex> & cache_lock)
+FileSegments LRUFileCache::splitRangeIntoCells(
+    const Key & key, size_t offset, size_t size, FileSegment::State state, std::lock_guard<std::mutex> & cache_lock)
 {
     assert(size > 0);
 
@@ -222,9 +221,10 @@ FileSegments LRUFileCache::splitRangeIntoEmptyCells(
         current_cell_size = std::min(remaining_size, max_file_segment_size);
         remaining_size -= current_cell_size;
 
-        auto * cell = addCell(key, current_pos, current_cell_size, FileSegment::State::EMPTY, cache_lock);
+        auto * cell = addCell(key, current_pos, current_cell_size, state, cache_lock);
         if (cell)
             file_segments.push_back(cell->file_segment);
+        assert(cell);
 
         current_pos += current_cell_size;
     }
@@ -250,7 +250,7 @@ FileSegmentsHolder LRUFileCache::getOrSet(const Key & key, size_t offset, size_t
 
     if (file_segments.empty())
     {
-        file_segments = splitRangeIntoEmptyCells(key, offset, size, cache_lock);
+        file_segments = splitRangeIntoCells(key, offset, size, FileSegment::State::EMPTY, cache_lock);
     }
     else
     {
@@ -295,7 +295,7 @@ FileSegmentsHolder LRUFileCache::getOrSet(const Key & key, size_t offset, size_t
             assert(current_pos < segment_range.left);
 
             auto hole_size = segment_range.left - current_pos;
-            file_segments.splice(it, splitRangeIntoEmptyCells(key, current_pos, hole_size, cache_lock));
+            file_segments.splice(it, splitRangeIntoCells(key, current_pos, hole_size, FileSegment::State::EMPTY, cache_lock));
 
             current_pos = segment_range.right + 1;
             ++it;
@@ -309,7 +309,7 @@ FileSegmentsHolder LRUFileCache::getOrSet(const Key & key, size_t offset, size_t
             /// segmentN
 
             auto hole_size = range.right - current_pos + 1;
-            file_segments.splice(file_segments.end(), splitRangeIntoEmptyCells(key, current_pos, hole_size, cache_lock));
+            file_segments.splice(file_segments.end(), splitRangeIntoCells(key, current_pos, hole_size, FileSegment::State::EMPTY, cache_lock));
         }
     }
 
@@ -354,6 +354,21 @@ LRUFileCache::FileSegmentCell * LRUFileCache::addCell(
     return &(it->second);
 }
 
+FileSegmentsHolder LRUFileCache::setDownloading(const Key & key, size_t offset, size_t size)
+{
+    std::lock_guard cache_lock(mutex);
+
+    auto * cell = getCell(key, offset, cache_lock);
+    if (cell)
+        throw Exception(
+            ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR,
+            "Cache cell already exists for key `{}` and offset {}",
+            keyToStr(key), offset);
+
+    auto file_segments = splitRangeIntoCells(key, offset, size, FileSegment::State::DOWNLOADING, cache_lock);
+    return FileSegmentsHolder(std::move(file_segments));
+}
+
 bool LRUFileCache::tryReserve(
     const Key & key_, size_t offset_, size_t size, std::lock_guard<std::mutex> & cache_lock)
 {
@@ -372,7 +387,8 @@ bool LRUFileCache::tryReserve(
 
     auto is_overflow = [&]
     {
-        return (current_size + size - removed_size > max_size)
+        /// max_size == 0 means unlimited cache size, max_element_size means unlimited number of cache elements.
+        return (max_size != 0 && current_size + size - removed_size > max_size)
             || (max_element_size != 0 && queue_size > max_element_size);
     };
 
@@ -484,6 +500,30 @@ void LRUFileCache::remove(const Key & key)
         fs::remove(key_path);
 }
 
+void LRUFileCache::tryRemoveAll()
+{
+    /// Try remove all cached files by cache_base_path.
+    /// Only releasable file segments are evicted.
+
+    std::lock_guard cache_lock(mutex);
+
+    for (auto it = queue.begin(); it != queue.end();)
+    {
+        auto & [key, offset] = *it++;
+
+        auto * cell = getCell(key, offset, cache_lock);
+        if (cell->releasable())
+        {
+            auto file_segment = cell->file_segment;
+            if (file_segment)
+            {
+                std::lock_guard<std::mutex> segment_lock(file_segment->mutex);
+                remove(file_segment->key(), file_segment->offset(), cache_lock, segment_lock);
+            }
+        }
+    }
+}
+
 void LRUFileCache::remove(
     Key key, size_t offset,
     std::lock_guard<std::mutex> & cache_lock, std::lock_guard<std::mutex> & /* segment_lock */)
@@ -668,6 +708,38 @@ bool LRUFileCache::isLastFileSegmentHolder(
     return cell->file_segment.use_count() == 2;
 }
 
+FileSegments LRUFileCache::getSnapshot() const
+{
+    std::lock_guard cache_lock(mutex);
+
+    FileSegments file_segments;
+
+    for (const auto & [key, cells_by_offset] : files)
+    {
+        for (const auto & [offset, cell] : cells_by_offset)
+            file_segments.push_back(FileSegment::getSnapshot(cell.file_segment, cache_lock));
+    }
+
+    return file_segments;
+}
+
+std::vector<String> LRUFileCache::tryGetCachePaths(const Key & key)
+{
+    std::lock_guard cache_lock(mutex);
+
+    std::vector<String> cache_paths;
+
+    const auto & cells_by_offset = files[key];
+
+    for (const auto & [offset, cell] : cells_by_offset)
+    {
+        if (cell.file_segment->state() == FileSegment::State::DOWNLOADED)
+            cache_paths.push_back(getPathInLocalCache(key, offset));
+    }
+
+    return cache_paths;
+}
+
 LRUFileCache::FileSegmentCell::FileSegmentCell(FileSegmentPtr file_segment_, LRUQueue & queue_)
     : file_segment(file_segment_)
 {
@@ -685,12 +757,13 @@ LRUFileCache::FileSegmentCell::FileSegmentCell(FileSegmentPtr file_segment_, LRU
             break;
         }
         case FileSegment::State::EMPTY:
+        case FileSegment::State::DOWNLOADING:
         {
             break;
         }
         default:
             throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR,
-                            "Can create cell with either DOWNLOADED or EMPTY state, got: {}",
+                            "Can create cell with either EMPTY, DOWNLOADED, DOWNLOADING state, got: {}",
                             FileSegment::stateToString(file_segment->download_state));
     }
 }

src/Common/FileCache.h

@@ -33,9 +33,7 @@ public:
 
     IFileCache(
         const String & cache_base_path_,
-        size_t max_size_,
-        size_t max_element_size_,
-        size_t max_file_segment_size_);
+        const FileCacheSettings & cache_settings_);
 
     virtual ~IFileCache() = default;
 
@@ -44,7 +42,9 @@ public:
 
     virtual void remove(const Key & key) = 0;
 
-    static bool shouldBypassCache();
+    virtual void tryRemoveAll() = 0;
+
+    static bool isReadOnly();
 
     /// Cache capacity in bytes.
     size_t capacity() const { return max_size; }
@@ -55,6 +55,10 @@ public:
 
     String getPathInLocalCache(const Key & key);
 
+    const String & getBasePath() const { return cache_base_path; }
+
+    virtual std::vector<String> tryGetCachePaths(const Key & key) = 0;
+
     /**
      * Given an `offset` and `size` representing [offset, offset + size) bytes interval,
      * return list of cached non-overlapping non-empty
@@ -68,6 +72,10 @@ public:
      */
     virtual FileSegmentsHolder getOrSet(const Key & key, size_t offset, size_t size) = 0;
 
+    virtual FileSegmentsHolder setDownloading(const Key & key, size_t offset, size_t size) = 0;
+
+    virtual FileSegments getSnapshot() const = 0;
+
     /// For debug.
     virtual String dumpStructure(const Key & key) = 0;
 
@@ -112,16 +120,22 @@ class LRUFileCache final : public IFileCache
 public:
     LRUFileCache(
         const String & cache_base_path_,
-        size_t max_size_,
-        size_t max_element_size_ = REMOTE_FS_OBJECTS_CACHE_DEFAULT_MAX_ELEMENTS,
-        size_t max_file_segment_size_ = REMOTE_FS_OBJECTS_CACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE);
+        const FileCacheSettings & cache_settings_);
 
     FileSegmentsHolder getOrSet(const Key & key, size_t offset, size_t size) override;
 
+    FileSegments getSnapshot() const override;
+
+    FileSegmentsHolder setDownloading(const Key & key, size_t offset, size_t size) override;
+
     void initialize() override;
 
     void remove(const Key & key) override;
 
+    void tryRemoveAll() override;
+
+    std::vector<String> tryGetCachePaths(const Key & key) override;
+
 private:
     using FileKeyAndOffset = std::pair<Key, size_t>;
     using LRUQueue = std::list<FileKeyAndOffset>;
@@ -194,8 +208,8 @@ private:
 
     void loadCacheInfoIntoMemory();
 
-    FileSegments splitRangeIntoEmptyCells(
-        const Key & key, size_t offset, size_t size, std::lock_guard<std::mutex> & cache_lock);
+    FileSegments splitRangeIntoCells(
+        const Key & key, size_t offset, size_t size, FileSegment::State state, std::lock_guard<std::mutex> & cache_lock);
 
     String dumpStructureImpl(const Key & key_, std::lock_guard<std::mutex> & cache_lock);
 

src/Common/FileCacheFactory.cpp

@@ -15,28 +15,53 @@ FileCacheFactory & FileCacheFactory::instance()
     return ret;
 }
 
-FileCachePtr FileCacheFactory::getImpl(const std::string & cache_base_path, std::lock_guard<std::mutex> &)
+FileCacheFactory::CacheByBasePath FileCacheFactory::getAll()
+{
+    std::lock_guard lock(mutex);
+    return caches;
+}
+
+const FileCacheSettings & FileCacheFactory::getSettings(const std::string & cache_base_path)
+{
+    std::lock_guard lock(mutex);
+
+    auto * cache_data = getImpl(cache_base_path, lock);
+    if (cache_data)
+        return cache_data->settings;
+
+    throw Exception(ErrorCodes::BAD_ARGUMENTS, "No cache found by path: {}", cache_base_path);
+}
+
+FileCacheFactory::CacheData * FileCacheFactory::getImpl(const std::string & cache_base_path, std::lock_guard<std::mutex> &)
 {
     auto it = caches.find(cache_base_path);
     if (it == caches.end())
         return nullptr;
-    return it->second;
+    return &it->second;
+}
+
+FileCachePtr FileCacheFactory::get(const std::string & cache_base_path)
+{
+    std::lock_guard lock(mutex);
+
+    auto * cache_data = getImpl(cache_base_path, lock);
+    if (cache_data)
+        return cache_data->cache;
+
+    throw Exception(ErrorCodes::BAD_ARGUMENTS, "No cache found by path: {}", cache_base_path);
 }
 
 FileCachePtr FileCacheFactory::getOrCreate(
-    const std::string & cache_base_path, size_t max_size, size_t max_elements_size, size_t max_file_segment_size)
+    const std::string & cache_base_path, const FileCacheSettings & file_cache_settings)
 {
     std::lock_guard lock(mutex);
-    auto cache = getImpl(cache_base_path, lock);
-    if (cache)
-    {
-        if (cache->capacity() != max_size)
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cache with path `{}` already exists, but has different max size", cache_base_path);
-        return cache;
-    }
 
-    cache = std::make_shared<LRUFileCache>(cache_base_path, max_size, max_elements_size, max_file_segment_size);
-    caches.emplace(cache_base_path, cache);
+    auto * cache_data = getImpl(cache_base_path, lock);
+    if (cache_data)
+        return cache_data->cache;
+
+    auto cache = std::make_shared<LRUFileCache>(cache_base_path, file_cache_settings);
+    caches.emplace(cache_base_path, CacheData(cache, file_cache_settings));
     return cache;
 }
 

src/Common/FileCacheFactory.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Common/FileCache_fwd.h>
+#include <Common/FileCacheSettings.h>
 
 #include <boost/noncopyable.hpp>
 #include <unordered_map>
@@ -14,16 +15,32 @@ namespace DB
  */
 class FileCacheFactory final : private boost::noncopyable
 {
+    struct CacheData
+    {
+        FileCachePtr cache;
+        FileCacheSettings settings;
+
+        CacheData(FileCachePtr cache_, const FileCacheSettings & settings_) : cache(cache_), settings(settings_) {}
+    };
+
+    using CacheByBasePath = std::unordered_map<std::string, CacheData>;
+
 public:
     static FileCacheFactory & instance();
 
-    FileCachePtr getOrCreate(const std::string & cache_base_path, size_t max_size, size_t max_elements_size, size_t max_file_segment_size);
+    FileCachePtr getOrCreate(const std::string & cache_base_path, const FileCacheSettings & file_cache_settings);
+
+    FileCachePtr get(const std::string & cache_base_path);
+
+    CacheByBasePath getAll();
+
+    const FileCacheSettings & getSettings(const std::string & cache_base_path);
 
 private:
-    FileCachePtr getImpl(const std::string & cache_base_path, std::lock_guard<std::mutex> &);
+    CacheData * getImpl(const std::string & cache_base_path, std::lock_guard<std::mutex> &);
 
     std::mutex mutex;
-    std::unordered_map<std::string, FileCachePtr> caches;
+    CacheByBasePath caches;
 };
 
 }

src/Common/FileCacheSettings.cpp

@@ -0,0 +1,16 @@
+#include "FileCacheSettings.h"
+
+#include <Poco/Util/AbstractConfiguration.h>
+
+namespace DB
+{
+
+void FileCacheSettings::loadFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix)
+{
+    max_size = config.getUInt64(config_prefix + ".data_cache_max_size", REMOTE_FS_OBJECTS_CACHE_DEFAULT_MAX_CACHE_SIZE);
+    max_elements = config.getUInt64(config_prefix + ".data_cache_max_elements", REMOTE_FS_OBJECTS_CACHE_DEFAULT_MAX_ELEMENTS);
+    max_file_segment_size = config.getUInt64(config_prefix + ".max_file_segment_size", REMOTE_FS_OBJECTS_CACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE);
+    cache_on_write_operations = config.getUInt64(config_prefix + ".cache_on_write_operations", false);
+}
+
+}

src/Common/FileCacheSettings.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <Common/FileCache_fwd.h>
+
+namespace Poco { namespace Util { class AbstractConfiguration; } }
+
+namespace DB
+{
+
+struct FileCacheSettings
+{
+    size_t max_size = 0;
+    size_t max_elements = REMOTE_FS_OBJECTS_CACHE_DEFAULT_MAX_ELEMENTS;
+    size_t max_file_segment_size = REMOTE_FS_OBJECTS_CACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE;
+    bool cache_on_write_operations = false;
+
+    void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix);
+};
+
+}

src/Common/FileCache_fwd.h

@@ -4,10 +4,13 @@
 namespace DB
 {
 
+static constexpr int REMOTE_FS_OBJECTS_CACHE_DEFAULT_MAX_CACHE_SIZE = 1024 * 1024 * 1024;
 static constexpr int REMOTE_FS_OBJECTS_CACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE = 100 * 1024 * 1024;
 static constexpr int REMOTE_FS_OBJECTS_CACHE_DEFAULT_MAX_ELEMENTS = 1024 * 1024;
 
 class IFileCache;
 using FileCachePtr = std::shared_ptr<IFileCache>;
 
+struct FileCacheSettings;
+
 }

src/Common/FileSegment.cpp

@@ -31,10 +31,34 @@ FileSegment::FileSegment(
     , log(&Poco::Logger::get("FileSegment"))
 #endif
 {
-    if (download_state == State::DOWNLOADED)
-        reserved_size = downloaded_size = size_;
-    else if (download_state != State::EMPTY)
-        throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Can create cell with either DOWNLOADED or EMPTY state");
+    /// On creation, file segment state can be EMPTY, DOWNLOADED, DOWNLOADING.
+    switch (download_state)
+    {
+        /// EMPTY is used when file segment is not in cache and
+        /// someone will _potentially_ want to download it (after calling getOrSetDownloader()).
+        case (State::EMPTY):
+        {
+            break;
+        }
+        /// DOWNLOADED is used either on initial cache metadata load into memory on server startup
+        /// or on reduceSizeToDownloaded() -- when file segment object is updated.
+        case (State::DOWNLOADED):
+        {
+            reserved_size = downloaded_size = size_;
+            break;
+        }
+        /// DOWNLOADING is used only for write-through caching (e.g. getOrSetDownloader() is not
+        /// needed, downloader is set on file segment creation).
+        case (State::DOWNLOADING):
+        {
+            downloader_id = getCallerId();
+            break;
+        }
+        default:
+        {
+            throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Can create cell with either EMPTY, DOWNLOADED, DOWNLOADING state");
+        }
+    }
 }
 
 FileSegment::State FileSegment::state() const
@@ -49,6 +73,12 @@ size_t FileSegment::getDownloadOffset() const
     return range().left + getDownloadedSize(segment_lock);
 }
 
+size_t FileSegment::getDownloadedSize() const
+{
+    std::lock_guard segment_lock(mutex);
+    return getDownloadedSize(segment_lock);
+}
+
 size_t FileSegment::getDownloadedSize(std::lock_guard<std::mutex> & /* segment_lock */) const
 {
     if (download_state == State::DOWNLOADED)
@@ -60,24 +90,15 @@ size_t FileSegment::getDownloadedSize(std::lock_guard<std::mutex> & /* segment_l
 
 String FileSegment::getCallerId()
 {
-    return getCallerIdImpl(false);
+    return getCallerIdImpl();
 }
 
-String FileSegment::getCallerIdImpl(bool allow_non_strict_checking)
+String FileSegment::getCallerIdImpl()
 {
-    if (IFileCache::shouldBypassCache())
-    {
-        /// getCallerId() can be called from completeImpl(), which can be called from complete().
-        /// complete() is called from destructor of CachedReadBufferFromRemoteFS when there is no query id anymore.
-        /// Allow non strict checking in this case. This works correctly as if getCallerIdImpl() is called from destructor,
-        /// then we know that caller is not a downloader, because downloader is reset each nextImpl() call either
-        /// manually or via SCOPE_EXIT.
-
-        if (allow_non_strict_checking)
-            return "None";
-
-        throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Cannot use cache without query id");
-    }
+    if (!CurrentThread::isInitialized()
+        || !CurrentThread::get().getQueryContext()
+        || CurrentThread::getQueryId().size == 0)
+        return "None:" + toString(getThreadId());
 
     return CurrentThread::getQueryId().toString() + ":" + toString(getThreadId());
 }
@@ -136,7 +157,6 @@ String FileSegment::getDownloader() const
 bool FileSegment::isDownloader() const
 {
     std::lock_guard segment_lock(mutex);
-    LOG_TEST(log, "Checking for current downloader. Caller: {}, downloader: {}, current state: {}", getCallerId(), downloader_id, stateToString(download_state));
     return getCallerId() == downloader_id;
 }
 
@@ -221,15 +241,9 @@ void FileSegment::write(const char * from, size_t size, size_t offset_)
     {
         std::lock_guard segment_lock(mutex);
 
-        auto info = getInfoForLogImpl(segment_lock);
-        e.addMessage("while writing into cache, info: " + info);
+        wrapWithCacheInfo(e, "while writing into cache", segment_lock);
 
-        LOG_ERROR(log, "Failed to write to cache. File segment info: {}", info);
-
-        download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION;
-
-        cache_writer->finalize();
-        cache_writer.reset();
+        setDownloadFailed(segment_lock);
 
         cv.notify_all();
 
@@ -239,6 +253,77 @@ void FileSegment::write(const char * from, size_t size, size_t offset_)
     assert(getDownloadOffset() == offset_ + size);
 }
 
+void FileSegment::writeInMemory(const char * from, size_t size)
+{
+    if (!size)
+        throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Attempt to write zero size cache file");
+
+    if (availableSize() < size)
+        throw Exception(
+            ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR,
+            "Not enough space is reserved. Available: {}, expected: {}", availableSize(), size);
+
+    std::lock_guard segment_lock(mutex);
+
+    if (cache_writer)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache writer already initialized");
+
+    auto download_path = cache->getPathInLocalCache(key(), offset());
+    cache_writer = std::make_unique<WriteBufferFromFile>(download_path, size + 1);
+
+    try
+    {
+        cache_writer->write(from, size);
+    }
+    catch (Exception & e)
+    {
+        wrapWithCacheInfo(e, "while writing into cache", segment_lock);
+
+        setDownloadFailed(segment_lock);
+
+        cv.notify_all();
+
+        throw;
+    }
+}
+
+size_t FileSegment::finalizeWrite()
+{
+    std::lock_guard segment_lock(mutex);
+
+    if (!cache_writer)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache writer not initialized");
+
+    size_t size = cache_writer->offset();
+
+    if (size == 0)
+        throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Writing size is not allowed");
+
+    try
+    {
+        cache_writer->next();
+    }
+    catch (Exception & e)
+    {
+        wrapWithCacheInfo(e, "while writing into cache", segment_lock);
+
+        setDownloadFailed(segment_lock);
+
+        cv.notify_all();
+
+        throw;
+    }
+
+    downloaded_size += size;
+
+    if (downloaded_size != range().size())
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected downloaded size to equal file segment size ({} == {})", downloaded_size, range().size());
+
+    setDownloaded(segment_lock);
+
+    return size;
+}
+
 FileSegment::State FileSegment::wait()
 {
     std::unique_lock segment_lock(mutex);
@@ -303,6 +388,20 @@ void FileSegment::setDownloaded(std::lock_guard<std::mutex> & /* segment_lock */
 {
     download_state = State::DOWNLOADED;
     is_downloaded = true;
+    downloader_id.clear();
+
+    if (cache_writer)
+    {
+        cache_writer->finalize();
+        cache_writer.reset();
+        remote_file_reader.reset();
+    }
+}
+
+void FileSegment::setDownloadFailed(std::lock_guard<std::mutex> & /* segment_lock */)
+{
+    download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION;
+    downloader_id.clear();
 
     if (cache_writer)
     {
@@ -360,7 +459,7 @@ void FileSegment::complete(State state)
     }
     catch (...)
     {
-        if (!downloader_id.empty() && downloader_id == getCallerIdImpl(true))
+        if (!downloader_id.empty() && downloader_id == getCallerIdImpl())
             downloader_id.clear();
 
         cv.notify_all();
@@ -385,7 +484,7 @@ void FileSegment::complete(std::lock_guard<std::mutex> & cache_lock)
         /// Segment state can be changed from DOWNLOADING or EMPTY only if the caller is the
         /// downloader or the only owner of the segment.
 
-        bool can_update_segment_state = downloader_id == getCallerIdImpl(true)
+        bool can_update_segment_state = downloader_id == getCallerIdImpl()
             || cache->isLastFileSegmentHolder(key(), offset(), cache_lock, segment_lock);
 
         if (can_update_segment_state)
@@ -394,11 +493,11 @@ void FileSegment::complete(std::lock_guard<std::mutex> & cache_lock)
 
     try
     {
-        completeImpl(cache_lock, segment_lock, /* allow_non_strict_checking */true);
+        completeImpl(cache_lock, segment_lock);
     }
     catch (...)
     {
-        if (!downloader_id.empty() && downloader_id == getCallerIdImpl(true))
+        if (!downloader_id.empty() && downloader_id == getCallerIdImpl())
             downloader_id.clear();
 
         cv.notify_all();
@@ -408,7 +507,7 @@ void FileSegment::complete(std::lock_guard<std::mutex> & cache_lock)
     cv.notify_all();
 }
 
-void FileSegment::completeImpl(std::lock_guard<std::mutex> & cache_lock, std::lock_guard<std::mutex> & segment_lock, bool allow_non_strict_checking)
+void FileSegment::completeImpl(std::lock_guard<std::mutex> & cache_lock, std::lock_guard<std::mutex> & segment_lock)
 {
     bool is_last_holder = cache->isLastFileSegmentHolder(key(), offset(), cache_lock, segment_lock);
 
@@ -444,7 +543,7 @@ void FileSegment::completeImpl(std::lock_guard<std::mutex> & cache_lock, std::lo
         }
     }
 
-    if (!downloader_id.empty() && (downloader_id == getCallerIdImpl(allow_non_strict_checking) || is_last_holder))
+    if (!downloader_id.empty() && (downloader_id == getCallerIdImpl() || is_last_holder))
     {
         LOG_TEST(log, "Clearing downloader id: {}, current state: {}", downloader_id, stateToString(download_state));
         downloader_id.clear();
@@ -471,6 +570,11 @@ String FileSegment::getInfoForLogImpl(std::lock_guard<std::mutex> & segment_lock
     return info.str();
 }
 
+void FileSegment::wrapWithCacheInfo(Exception & e, const String & message, std::lock_guard<std::mutex> & segment_lock) const
+{
+    e.addMessage(fmt::format("{}, current cache state: {}", message, getInfoForLogImpl(segment_lock)));
+}
+
 String FileSegment::stateToString(FileSegment::State state)
 {
     switch (state)
@@ -504,6 +608,23 @@ void FileSegment::assertCorrectnessImpl(std::lock_guard<std::mutex> & /* segment
     assert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(cache->getPathInLocalCache(key(), offset())) > 0);
 }
 
+FileSegmentPtr FileSegment::getSnapshot(const FileSegmentPtr & file_segment, std::lock_guard<std::mutex> & /* cache_lock */)
+{
+    auto snapshot = std::make_shared<FileSegment>(
+        file_segment->offset(),
+        file_segment->range().size(),
+        file_segment->key(),
+        nullptr,
+        State::EMPTY);
+
+    snapshot->hits_count = file_segment->getHitsCount();
+    snapshot->ref_count = file_segment.use_count();
+    snapshot->downloaded_size = file_segment->getDownloadedSize();
+    snapshot->download_state = file_segment->state();
+
+    return snapshot;
+}
+
 FileSegmentsHolder::~FileSegmentsHolder()
 {
     /// In CacheableReadBufferFromRemoteFS file segment's downloader removes file segments from

src/Common/FileSegment.h

@@ -97,6 +97,15 @@ public:
 
     void write(const char * from, size_t size, size_t offset_);
 
+    /**
+     * writeInMemory and finalizeWrite are used together to write a single file with delay.
+     * Both can be called only once, one after another. Used for writing cache via threadpool
+     * on wrote operations. TODO: this solution is temporary, until adding a separate cache layer.
+     */
+    void writeInMemory(const char * from, size_t size);
+
+    size_t finalizeWrite();
+
     RemoteFileReaderPtr getRemoteFileReader();
 
     void setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_);
@@ -117,14 +126,24 @@ public:
 
     size_t getDownloadOffset() const;
 
+    size_t getDownloadedSize() const;
+
     void completeBatchAndResetDownloader();
 
     void complete(State state);
 
     String getInfoForLog() const;
 
+    size_t getHitsCount() const { return hits_count; }
+
+    size_t getRefCount() const { return ref_count; }
+
+    void incrementHitsCount() { ++hits_count; }
+
     void assertCorrectness() const;
 
+    static FileSegmentPtr getSnapshot(const FileSegmentPtr & file_segment, std::lock_guard<std::mutex> & cache_lock);
+
 private:
     size_t availableSize() const { return reserved_size - downloaded_size; }
 
@@ -133,6 +152,9 @@ private:
     void assertCorrectnessImpl(std::lock_guard<std::mutex> & segment_lock) const;
 
     void setDownloaded(std::lock_guard<std::mutex> & segment_lock);
+    void setDownloadFailed(std::lock_guard<std::mutex> & segment_lock);
+
+    void wrapWithCacheInfo(Exception & e, const String & message, std::lock_guard<std::mutex> & segment_lock) const;
 
     bool lastFileSegmentHolder() const;
 
@@ -144,9 +166,9 @@ private:
 
     void completeImpl(
         std::lock_guard<std::mutex> & cache_lock,
-        std::lock_guard<std::mutex> & segment_lock, bool allow_non_strict_checking = false);
+        std::lock_guard<std::mutex> & segment_lock);
 
-    static String getCallerIdImpl(bool allow_non_strict_checking = false);
+    static String getCallerIdImpl();
 
     void resetDownloaderImpl(std::lock_guard<std::mutex> & segment_lock);
 
@@ -180,6 +202,8 @@ private:
     bool detached = false;
 
     std::atomic<bool> is_downloaded{false};
+    std::atomic<size_t> hits_count = 0; /// cache hits.
+    std::atomic<size_t> ref_count = 0; /// Used for getting snapshot state
 };
 
 struct FileSegmentsHolder : private boost::noncopyable

src/Common/ProfileEvents.cpp

@@ -297,6 +297,25 @@
     M(MergeTreeMetadataCacheHit, "Number of times the read of meta file was done from MergeTree metadata cache") \
     M(MergeTreeMetadataCacheMiss, "Number of times the read of meta file was not done from MergeTree metadata cache") \
     \
+    M(KafkaRebalanceRevocations, "Number of partition revocations (the first stage of consumer group rebalance)") \
+    M(KafkaRebalanceAssignments, "Number of partition assignments (the final stage of consumer group rebalance)") \
+    M(KafkaRebalanceErrors, "Number of failed consumer group rebalances") \
+    M(KafkaMessagesPolled, "Number of Kafka messages polled from librdkafka to ClickHouse") \
+    M(KafkaMessagesRead, "Number of Kafka messages already processed by ClickHouse") \
+    M(KafkaMessagesFailed, "Number of Kafka messages ClickHouse failed to parse") \
+    M(KafkaRowsRead, "Number of rows parsed from Kafka messages") \
+    M(KafkaRowsRejected, "Number of parsed rows which were later rejected (due to rebalances / errors or similar reasons). Those rows will be consumed again after the rebalance.") \
+    M(KafkaDirectReads, "Number of direct selects from Kafka tables since server start") \
+    M(KafkaBackgroundReads, "Number of background reads populating materialized views from Kafka since server start") \
+    M(KafkaCommits, "Number of successful commits of consumed offsets to Kafka (normally should be the same as KafkaBackgroundReads)") \
+    M(KafkaCommitFailures, "Number of failed commits of consumed offsets to Kafka (usually is a sign of some data duplication)") \
+    M(KafkaConsumerErrors, "Number of errors reported by librdkafka during polls") \
+    M(KafkaWrites, "Number of writes (inserts) to Kafka tables ") \
+    M(KafkaRowsWritten, "Number of rows inserted into Kafka tables") \
+    M(KafkaProducerFlushes, "Number of explicit flushes to Kafka producer") \
+    M(KafkaMessagesProduced, "Number of messages produced to Kafka") \
+    M(KafkaProducerErrors, "Number of errors during producing the messages to Kafka") \
+    \
     M(ScalarSubqueriesGlobalCacheHit, "Number of times a read from a scalar subquery was done using the global cache") \
     M(ScalarSubqueriesLocalCacheHit, "Number of times a read from a scalar subquery was done using the local cache") \
     M(ScalarSubqueriesCacheMiss, "Number of times a read from a scalar subquery was not cached and had to be calculated completely")

src/Common/SystemLogBase.cpp

@@ -9,7 +9,9 @@
 #include <Interpreters/SessionLog.h>
 #include <Interpreters/TextLog.h>
 #include <Interpreters/TraceLog.h>
+#include <Interpreters/ProcessorsProfileLog.h>
 #include <Interpreters/ZooKeeperLog.h>
+#include <Interpreters/TransactionsInfoLog.h>
 
 #include <Common/MemoryTrackerBlockerInThread.h>
 #include <Common/SystemLogBase.h>

src/Common/SystemLogBase.h

@@ -23,7 +23,9 @@
     M(QueryViewsLogElement) \
     M(SessionLogElement) \
     M(TraceLogElement) \
+    M(TransactionsInfoLogElement) \
     M(ZooKeeperLogElement) \
+    M(ProcessorProfileLogElement) \
     M(TextLogElement)
 
 namespace Poco

src/Common/ThreadStatus.h

@@ -216,6 +216,11 @@ public:
         return query_context.lock();
     }
 
+    auto getGlobalContext() const
+    {
+        return global_context.lock();
+    }
+
     void disableProfiling()
     {
         assert(!query_profiler_real && !query_profiler_cpu);

src/Common/TransactionID.cpp

@@ -0,0 +1,43 @@
+#include <Common/TransactionID.h>
+#include <Common/SipHash.h>
+#include <IO/ReadHelpers.h>
+#include <IO/WriteHelpers.h>
+
+namespace DB
+{
+
+TIDHash TransactionID::getHash() const
+{
+    SipHash hash;
+    hash.update(start_csn);
+    hash.update(local_tid);
+    hash.update(host_id);
+    return hash.get64();
+}
+
+
+void TransactionID::write(const TransactionID & tid, WriteBuffer & buf)
+{
+    writeChar('(', buf);
+    writeText(tid.start_csn, buf);
+    writeCString(", ", buf);
+    writeText(tid.local_tid, buf);
+    writeCString(", ", buf);
+    writeText(tid.host_id, buf);
+    writeChar(')', buf);
+}
+
+TransactionID TransactionID::read(ReadBuffer & buf)
+{
+    TransactionID tid = Tx::EmptyTID;
+    assertChar('(', buf);
+    readText(tid.start_csn, buf);
+    assertString(", ", buf);
+    readText(tid.local_tid, buf);
+    assertString(", ", buf);
+    readText(tid.host_id, buf);
+    assertChar(')', buf);
+    return tid;
+}
+
+}

src/Common/TransactionID.h

@@ -0,0 +1,115 @@
+#pragma once
+#include <Core/Types.h>
+#include <Core/UUID.h>
+#include <fmt/format.h>
+#include <IO/WriteHelpers.h>
+
+namespace DB
+{
+
+class IDataType;
+using DataTypePtr = std::shared_ptr<const IDataType>;
+class MergeTreeTransaction;
+
+/// This macro is useful for places where a pointer to current transaction should be passed,
+/// but transactions are not supported yet (e.g. when calling MergeTreeData's methods from StorageReplicatedMergeTree)
+/// or transaction object is not needed and not passed intentionally.
+#ifndef NO_TRANSACTION_PTR
+#define NO_TRANSACTION_PTR std::shared_ptr<MergeTreeTransaction>(nullptr)
+#define NO_TRANSACTION_RAW static_cast<MergeTreeTransaction *>(nullptr)
+#endif
+
+/// Commit Sequence Number
+using CSN = UInt64;
+/// Local part of TransactionID
+using LocalTID = UInt64;
+/// Hash of TransactionID that fits into 64-bit atomic
+using TIDHash = UInt64;
+
+namespace Tx
+{
+    /// For transactions that are probably not committed (yet)
+    const CSN UnknownCSN = 0;
+    /// For changes were made without creating a transaction
+    const CSN PrehistoricCSN = 1;
+    /// Special reserved values
+    const CSN CommittingCSN = 2;
+    const CSN EverythingVisibleCSN = 3;
+    const CSN MaxReservedCSN = 32;
+
+    /// So far, that changes will never become visible
+    const CSN RolledBackCSN = std::numeric_limits<CSN>::max();
+
+    const LocalTID PrehistoricLocalTID = 1;
+    const LocalTID DummyLocalTID = 2;
+    const LocalTID MaxReservedLocalTID = 32;
+}
+
+struct TransactionID
+{
+    /// Global sequential number, the newest commit timestamp the we saw when this transaction began
+    CSN start_csn = 0;
+    /// Local sequential that is unique for each transaction started by this host within specific start_csn
+    LocalTID local_tid = 0;
+    /// UUID of host that has started this transaction
+    UUID host_id = UUIDHelpers::Nil;
+
+    /// NOTE Maybe we could just generate UUIDv4 for each transaction, but it would be harder to debug.
+    /// Partial order is defined for this TransactionID structure:
+    /// (tid1.start_csn <= tid2.start_csn)    <==>    (tid1 <= tid2)
+    /// (tid1.start_csn == tid2.start_csn && tid1.host_id == tid2.host_id && tid1.local_tid < tid2.local_tid)    ==>    (tid1 < tid2)
+    /// If two transaction have the same start_csn, but were started by different hosts, then order is undefined.
+
+    bool operator == (const TransactionID & rhs) const
+    {
+        return start_csn == rhs.start_csn && local_tid == rhs.local_tid && host_id == rhs.host_id;
+    }
+
+    bool operator != (const TransactionID & rhs) const
+    {
+        return !(*this == rhs);
+    }
+
+    TIDHash getHash() const;
+
+    bool isEmpty() const
+    {
+        assert((local_tid == 0) == (start_csn == 0 && host_id == UUIDHelpers::Nil));
+        return local_tid == 0;
+    }
+
+    bool isPrehistoric() const
+    {
+        assert((local_tid == Tx::PrehistoricLocalTID) == (start_csn == Tx::PrehistoricCSN));
+        return local_tid == Tx::PrehistoricLocalTID;
+    }
+
+
+    static void write(const TransactionID & tid, WriteBuffer & buf);
+    static TransactionID read(ReadBuffer & buf);
+};
+
+namespace Tx
+{
+    const TransactionID EmptyTID = {0, 0, UUIDHelpers::Nil};
+    const TransactionID PrehistoricTID = {PrehistoricCSN, PrehistoricLocalTID, UUIDHelpers::Nil};
+    const TransactionID DummyTID = {PrehistoricCSN, DummyLocalTID, UUIDHelpers::Nil};
+}
+
+}
+
+template<>
+struct fmt::formatter<DB::TransactionID>
+{
+    template<typename ParseContext>
+    constexpr auto parse(ParseContext & context)
+    {
+        return context.begin();
+    }
+
+    template<typename FormatContext>
+    auto format(const DB::TransactionID & tid, FormatContext & context)
+    {
+        return fmt::format_to(context.out(), "({}, {}, {})", tid.start_csn, tid.local_tid, tid.host_id);
+    }
+};

src/Common/ZooKeeper/ZooKeeper.cpp

@@ -701,24 +701,34 @@ void ZooKeeper::removeChildrenRecursive(const std::string & path, const String &
     }
 }
 
-void ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, const String & keep_child_node)
+bool ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, bool probably_flat, const String & keep_child_node)
 {
     Strings children;
     if (tryGetChildren(path, children) != Coordination::Error::ZOK)
-        return;
+        return false;
+
+    bool removed_as_expected = true;
     while (!children.empty())
     {
         Coordination::Requests ops;
         Strings batch;
+        ops.reserve(MULTI_BATCH_SIZE);
+        batch.reserve(MULTI_BATCH_SIZE);
         for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i)
         {
             String child_path = fs::path(path) / children.back();
-            tryRemoveChildrenRecursive(child_path);
+
+            /// Will try to avoid recursive getChildren calls if child_path probably has no children.
+            /// It may be extremely slow when path contain a lot of leaf children.
+            if (!probably_flat)
+                tryRemoveChildrenRecursive(child_path);
+
             if (likely(keep_child_node.empty() || keep_child_node != children.back()))
             {
                 batch.push_back(child_path);
                 ops.emplace_back(zkutil::makeRemoveRequest(child_path, -1));
             }
+
             children.pop_back();
         }
 
@@ -726,10 +736,39 @@ void ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, const Strin
         /// this means someone is concurrently removing these children and we will have
         /// to remove them one by one.
         Coordination::Responses responses;
-        if (tryMulti(ops, responses) != Coordination::Error::ZOK)
-            for (const std::string & child : batch)
-                tryRemove(child);
+        if (tryMulti(ops, responses) == Coordination::Error::ZOK)
+            continue;
+
+        removed_as_expected = false;
+
+        std::vector<zkutil::ZooKeeper::FutureRemove> futures;
+        futures.reserve(batch.size());
+        for (const std::string & child : batch)
+            futures.push_back(asyncTryRemoveNoThrow(child, -1));
+
+        for (size_t i = 0; i < batch.size(); ++i)
+        {
+            auto res = futures[i].get();
+            if (res.error == Coordination::Error::ZOK)
+                continue;
+            if (res.error == Coordination::Error::ZNONODE)
+                continue;
+
+            if (res.error == Coordination::Error::ZNOTEMPTY)
+            {
+                if (probably_flat)
+                {
+                    /// It actually has children, let's remove them
+                    tryRemoveChildrenRecursive(batch[i]);
+                    tryRemove(batch[i]);
+                }
+                continue;
+            }
+
+            throw KeeperException(res.error, batch[i]);
+        }
     }
+    return removed_as_expected;
 }
 
 void ZooKeeper::removeRecursive(const std::string & path)
@@ -1231,4 +1270,14 @@ String extractZooKeeperPath(const String & path, bool check_starts_with_slash, P
     return normalizeZooKeeperPath(path, check_starts_with_slash, log);
 }
 
+String getSequentialNodeName(const String & prefix, UInt64 number)
+{
+    /// NOTE Sequential counter in ZooKeeper is Int32.
+    assert(number < std::numeric_limits<Int32>::max());
+    constexpr size_t seq_node_digits = 10;
+    String num_str = std::to_string(number);
+    String name = prefix + String(seq_node_digits - num_str.size(), '0') + num_str;
+    return name;
+}
+
 }

src/Common/ZooKeeper/ZooKeeper.h

@@ -225,7 +225,10 @@ public:
     /// If keep_child_node is not empty, this method will not remove path/keep_child_node (but will remove its subtree).
     /// It can be useful to keep some child node as a flag which indicates that path is currently removing.
     void removeChildrenRecursive(const std::string & path, const String & keep_child_node = {});
-    void tryRemoveChildrenRecursive(const std::string & path, const String & keep_child_node = {});
+    /// If probably_flat is true, this method will optimistically try to remove children non-recursive
+    /// and will fall back to recursive removal if it gets ZNOTEMPTY for some child.
+    /// Returns true if no kind of fallback happened.
+    bool tryRemoveChildrenRecursive(const std::string & path, bool probably_flat = false, const String & keep_child_node = {});
 
     /// Remove all children nodes (non recursive).
     void removeChildren(const std::string & path);
@@ -414,4 +417,6 @@ String extractZooKeeperName(const String & path);
 
 String extractZooKeeperPath(const String & path, bool check_starts_with_slash, Poco::Logger * log = nullptr);
 
+String getSequentialNodeName(const String & prefix, UInt64 number);
+
 }

src/Common/ZooKeeper/ZooKeeperImpl.cpp

@@ -846,7 +846,7 @@ void ZooKeeper::receiveEvent()
 void ZooKeeper::finalize(bool error_send, bool error_receive, const String & reason)
 {
     /// If some thread (send/receive) already finalizing session don't try to do it
-    bool already_started = finalization_started.exchange(true);
+    bool already_started = finalization_started.test_and_set();
 
     LOG_TEST(log, "Finalizing session {}: finalization_started={}, queue_finished={}, reason={}",
              session_id, already_started, requests_queue.isFinished(), reason);

src/Common/ZooKeeper/ZooKeeperImpl.h

@@ -209,7 +209,7 @@ private:
     std::atomic<XID> next_xid {1};
     /// Mark session finalization start. Used to avoid simultaneous
     /// finalization from different threads. One-shot flag.
-    std::atomic<bool> finalization_started {false};
+    std::atomic_flag finalization_started;
 
     using clock = std::chrono::steady_clock;
 

src/Common/tests/gtest_lru_file_cache.cpp

@@ -4,6 +4,7 @@
 #include <Common/FileCache.h>
 #include <Common/CurrentThread.h>
 #include <Common/filesystemHelpers.h>
+#include <Common/FileCacheSettings.h>
 #include <Common/tests/gtest_global_context.h>
 #include <Common/SipHash.h>
 #include <Common/hex.h>
@@ -102,7 +103,10 @@ TEST(LRUFileCache, get)
     query_context->setCurrentQueryId("query_id");
     DB::CurrentThread::QueryScope query_scope_holder(query_context);
 
-    auto cache = DB::LRUFileCache(cache_base_path, 30, 5);
+    DB::FileCacheSettings settings;
+    settings.max_size = 30;
+    settings.max_elements = 5;
+    auto cache = DB::LRUFileCache(cache_base_path, settings);
     cache.initialize();
     auto key = cache.hash("key1");
 
@@ -472,7 +476,7 @@ TEST(LRUFileCache, get)
     {
         /// Test LRUCache::restore().
 
-        auto cache2 = DB::LRUFileCache(cache_base_path, 30, 5);
+        auto cache2 = DB::LRUFileCache(cache_base_path, settings);
         cache2.initialize();
 
         ASSERT_EQ(cache2.getStat().downloaded_size, 5);
@@ -491,7 +495,9 @@ TEST(LRUFileCache, get)
     {
         /// Test max file segment size
 
-        auto cache2 = DB::LRUFileCache(caches_dir / "cache2", 30, 5, /* max_file_segment_size */10);
+        auto settings2 = settings;
+        settings2.max_file_segment_size = 10;
+        auto cache2 = DB::LRUFileCache(caches_dir / "cache2", settings2);
         cache2.initialize();
 
         auto holder1 = cache2.getOrSet(key, 0, 25); /// Get [0, 24]

src/Coordination/Changelog.cpp

@@ -222,8 +222,8 @@ public:
                 }
 
                 /// Check for duplicated changelog ids
-                if (logs.count(record.header.index) != 0)
-                    std::erase_if(logs, [record] (const auto & item) { return item.first >= record.header.index; });
+                if (logs.contains(record.header.index))
+                    std::erase_if(logs, [&record] (const auto & item) { return item.first >= record.header.index; });
 
                 result.total_entries_read_from_log += 1;
 
@@ -659,6 +659,7 @@ LogEntryPtr Changelog::getLatestConfigChange() const
 nuraft::ptr<nuraft::buffer> Changelog::serializeEntriesToBuffer(uint64_t index, int32_t count)
 {
     std::vector<nuraft::ptr<nuraft::buffer>> returned_logs;
+    returned_logs.reserve(count);
 
     uint64_t size_total = 0;
     for (uint64_t i = index; i < index + count; ++i)
@@ -669,7 +670,7 @@ nuraft::ptr<nuraft::buffer> Changelog::serializeEntriesToBuffer(uint64_t index,
 
         nuraft::ptr<nuraft::buffer> buf = entry->second->serialize();
         size_total += buf->size();
-        returned_logs.push_back(buf);
+        returned_logs.push_back(std::move(buf));
     }
 
     nuraft::ptr<nuraft::buffer> buf_out = nuraft::buffer::alloc(sizeof(int32_t) + count * sizeof(int32_t) + size_total);
@@ -678,9 +679,8 @@ nuraft::ptr<nuraft::buffer> Changelog::serializeEntriesToBuffer(uint64_t index,
 
     for (auto & entry : returned_logs)
     {
-        nuraft::ptr<nuraft::buffer> & bb = entry;
-        buf_out->put(static_cast<int32_t>(bb->size()));
-        buf_out->put(*bb);
+        buf_out->put(static_cast<int32_t>(entry->size()));
+        buf_out->put(*entry);
     }
     return buf_out;
 }
@@ -699,7 +699,7 @@ void Changelog::applyEntriesFromBuffer(uint64_t index, nuraft::buffer & buffer)
         buffer.get(buf_local);
 
         LogEntryPtr log_entry = nuraft::log_entry::deserialize(*buf_local);
-        if (i == 0 && logs.count(cur_index))
+        if (i == 0 && logs.contains(cur_index))
             writeAt(cur_index, log_entry);
         else
             appendEntry(cur_index, log_entry);

src/Coordination/KeeperDispatcher.cpp

@@ -121,7 +121,7 @@ void KeeperDispatcher::requestThread()
                         current_batch.clear();
                     }
 
-                    prev_batch = current_batch;
+                    prev_batch = std::move(current_batch);
                     prev_result = result;
                 }
 

src/Coordination/KeeperSnapshotManager.cpp

@@ -43,7 +43,7 @@ namespace
 
     void writeNode(const KeeperStorage::Node & node, SnapshotVersion version, WriteBuffer & out)
     {
-        writeBinary(node.data, out);
+        writeBinary(node.getData(), out);
 
         /// Serialize ACL
         writeBinary(node.acl_id, out);
@@ -71,7 +71,9 @@ namespace
 
     void readNode(KeeperStorage::Node & node, ReadBuffer & in, SnapshotVersion version, ACLMap & acl_map)
     {
-        readBinary(node.data, in);
+        String new_data;
+        readBinary(new_data, in);
+        node.setData(std::move(new_data));
 
         if (version >= SnapshotVersion::V1)
         {
@@ -281,7 +283,7 @@ void KeeperStorageSnapshot::deserialize(SnapshotDeserializationResult & deserial
         if (itr.key != "/")
         {
             auto parent_path = parentPath(itr.key);
-            storage.container.updateValue(parent_path, [path = itr.key] (KeeperStorage::Node & value) { value.children.insert(getBaseName(path)); });
+            storage.container.updateValue(parent_path, [path = itr.key] (KeeperStorage::Node & value) { value.addChild(getBaseName(path)); });
         }
     }
 

src/Coordination/KeeperStateMachine.cpp

@@ -259,22 +259,9 @@ void KeeperStateMachine::save_logical_snp_obj(
 {
     LOG_DEBUG(log, "Saving snapshot {} obj_id {}", s.get_last_log_idx(), obj_id);
 
-    nuraft::ptr<nuraft::buffer> cloned_buffer;
-    nuraft::ptr<nuraft::snapshot> cloned_meta;
-    if (obj_id == 0) /// Fake snapshot required by NuRaft at startup
-    {
-        std::lock_guard lock(storage_and_responses_lock);
-        KeeperStorageSnapshot snapshot(storage.get(), s.get_last_log_idx(), getClusterConfig());
-        cloned_buffer = snapshot_manager.serializeSnapshotToBuffer(snapshot);
-    }
-    else
-    {
-        /// copy snapshot into memory
-    }
-
     /// copy snapshot meta into memory
     nuraft::ptr<nuraft::buffer> snp_buf = s.serialize();
-    cloned_meta = nuraft::snapshot::deserialize(*snp_buf);
+    nuraft::ptr<nuraft::snapshot> cloned_meta = nuraft::snapshot::deserialize(*snp_buf);
 
     try
     {
@@ -332,31 +319,22 @@ int KeeperStateMachine::read_logical_snp_obj(
 {
 
     LOG_DEBUG(log, "Reading snapshot {} obj_id {}", s.get_last_log_idx(), obj_id);
-    if (obj_id == 0) /// Fake snapshot required by NuRaft at startup
+
+    std::lock_guard lock(snapshots_lock);
+    /// Our snapshot is not equal to required. Maybe we still creating it in the background.
+    /// Let's wait and NuRaft will retry this call.
+    if (s.get_last_log_idx() != latest_snapshot_meta->get_last_log_idx())
     {
-        data_out = nuraft::buffer::alloc(sizeof(int32_t));
-        nuraft::buffer_serializer bs(data_out);
-        bs.put_i32(0);
-        is_last_obj = false;
+        LOG_WARNING(log, "Required to apply snapshot with last log index {}, but our last log index is {}. Will ignore this one and retry",
+                        s.get_last_log_idx(), latest_snapshot_meta->get_last_log_idx());
+        return -1;
     }
-    else
+    if (bufferFromFile(log, latest_snapshot_path, data_out))
     {
-        std::lock_guard lock(snapshots_lock);
-        /// Our snapshot is not equal to required. Maybe we still creating it in the background.
-        /// Let's wait and NuRaft will retry this call.
-        if (s.get_last_log_idx() != latest_snapshot_meta->get_last_log_idx())
-        {
-            LOG_WARNING(log, "Required to apply snapshot with last log index {}, but our last log index is {}. Will ignore this one and retry",
-                            s.get_last_log_idx(), latest_snapshot_meta->get_last_log_idx());
-            return -1;
-        }
-        if (bufferFromFile(log, latest_snapshot_path, data_out))
-        {
-            LOG_WARNING(log, "Error reading snapshot {} from {}", s.get_last_log_idx(), latest_snapshot_path);
-            return -1;
-        }
-        is_last_obj = true;
+        LOG_WARNING(log, "Error reading snapshot {} from {}", s.get_last_log_idx(), latest_snapshot_path);
+        return -1;
     }
+    is_last_obj = true;
 
     return 1;
 }

src/Coordination/KeeperStateManager.h

@@ -84,7 +84,7 @@ public:
     bool shouldStartAsFollower() const
     {
         std::lock_guard lock(configuration_wrapper_mutex);
-        return configuration_wrapper.servers_start_as_followers.count(my_server_id);
+        return configuration_wrapper.servers_start_as_followers.contains(my_server_id);
     }
 
     bool isSecure() const

src/Coordination/KeeperStorage.cpp

@@ -24,7 +24,10 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
 }
 
-static String base64Encode(const String & decoded)
+namespace
+{
+
+String base64Encode(const String & decoded)
 {
     std::ostringstream ostr; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
     ostr.exceptions(std::ios::failbit);
@@ -35,7 +38,7 @@ static String base64Encode(const String & decoded)
     return ostr.str();
 }
 
-static String getSHA1(const String & userdata)
+String getSHA1(const String & userdata)
 {
     Poco::SHA1Engine engine;
     engine.update(userdata);
@@ -43,14 +46,14 @@ static String getSHA1(const String & userdata)
     return String{digest_id.begin(), digest_id.end()};
 }
 
-static String generateDigest(const String & userdata)
+String generateDigest(const String & userdata)
 {
     std::vector<String> user_password;
     boost::split(user_password, userdata, [](char c) { return c == ':'; });
     return user_password[0] + ":" + base64Encode(getSHA1(userdata));
 }
 
-static bool checkACL(int32_t permission, const Coordination::ACLs & node_acls, const std::vector<KeeperStorage::AuthID> & session_auths)
+bool checkACL(int32_t permission, const Coordination::ACLs & node_acls, const std::vector<KeeperStorage::AuthID> & session_auths)
 {
     if (node_acls.empty())
         return true;
@@ -77,7 +80,7 @@ static bool checkACL(int32_t permission, const Coordination::ACLs & node_acls, c
     return false;
 }
 
-static bool fixupACL(
+bool fixupACL(
     const std::vector<Coordination::ACL> & request_acls,
     const std::vector<KeeperStorage::AuthID> & current_ids,
     std::vector<Coordination::ACL> & result_acls)
@@ -119,7 +122,7 @@ static bool fixupACL(
     return valid_found;
 }
 
-static KeeperStorage::ResponsesForSessions processWatchesImpl(const String & path, KeeperStorage::Watches & watches, KeeperStorage::Watches & list_watches, Coordination::Event event_type)
+KeeperStorage::ResponsesForSessions processWatchesImpl(const String & path, KeeperStorage::Watches & watches, KeeperStorage::Watches & list_watches, Coordination::Event event_type)
 {
     KeeperStorage::ResponsesForSessions result;
     auto it = watches.find(path);
@@ -174,6 +177,25 @@ static KeeperStorage::ResponsesForSessions processWatchesImpl(const String & pat
     }
     return result;
 }
+}
+
+void KeeperStorage::Node::setData(String new_data)
+{
+    size_bytes = size_bytes - data.size() + new_data.size();
+    data = std::move(new_data);
+}
+
+void KeeperStorage::Node::addChild(StringRef child_path)
+{
+    size_bytes += sizeof child_path;
+    children.insert(child_path);
+}
+
+void KeeperStorage::Node::removeChild(StringRef child_path)
+{
+    size_bytes -= sizeof child_path;
+    children.erase(child_path);
+}
 
 KeeperStorage::KeeperStorage(int64_t tick_time_ms, const String & superdigest_)
     : session_expiry_queue(tick_time_ms)
@@ -314,8 +336,8 @@ struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestPr
         created_node.stat.numChildren = 0;
         created_node.stat.dataLength = request.data.length();
         created_node.stat.ephemeralOwner = request.is_ephemeral ? session_id : 0;
-        created_node.data = request.data;
         created_node.is_sequental = request.is_sequential;
+        created_node.setData(std::move(request.data));
 
         auto [map_key, _] = container.insert(path_created, created_node);
         /// Take child path from key owned by map.
@@ -327,8 +349,7 @@ struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestPr
         container.updateValue(parent_path, [child_path, zxid, &prev_parent_zxid,
                                             parent_cversion, &prev_parent_cversion] (KeeperStorage::Node & parent)
         {
-            parent.children.insert(child_path);
-            parent.size_bytes += child_path.size;
+            parent.addChild(child_path);
             prev_parent_cversion = parent.stat.cversion;
             prev_parent_zxid = parent.stat.pzxid;
 
@@ -363,8 +384,7 @@ struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestPr
                 --undo_parent.seq_num;
                 undo_parent.stat.cversion = prev_parent_cversion;
                 undo_parent.stat.pzxid = prev_parent_zxid;
-                undo_parent.children.erase(child_path);
-                undo_parent.size_bytes -= child_path.size;
+                undo_parent.removeChild(child_path);
             });
 
             storage.container.erase(path_created);
@@ -409,7 +429,7 @@ struct KeeperStorageGetRequestProcessor final : public KeeperStorageRequestProce
         else
         {
             response.stat = it->value.stat;
-            response.data = it->value.data;
+            response.data = it->value.getData();
             response.error = Coordination::Error::ZOK;
         }
 
@@ -498,8 +518,7 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr
             {
                 --parent.stat.numChildren;
                 ++parent.stat.cversion;
-                parent.children.erase(child_basename);
-                parent.size_bytes -= child_basename.size;
+                parent.removeChild(child_basename);
             });
 
             response.error = Coordination::Error::ZOK;
@@ -520,8 +539,7 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr
                 {
                     ++parent.stat.numChildren;
                     --parent.stat.cversion;
-                    parent.children.insert(child_name);
-                    parent.size_bytes += child_name.size;
+                    parent.addChild(child_name);
                 });
             };
         }
@@ -598,14 +616,13 @@ struct KeeperStorageSetRequestProcessor final : public KeeperStorageRequestProce
 
             auto prev_node = it->value;
 
-            auto itr = container.updateValue(request.path, [zxid, request, time] (KeeperStorage::Node & value)
+            auto itr = container.updateValue(request.path, [zxid, request, time] (KeeperStorage::Node & value) mutable
             {
                 value.stat.version++;
                 value.stat.mzxid = zxid;
                 value.stat.mtime = time;
                 value.stat.dataLength = request.data.length();
-                value.size_bytes = value.size_bytes + request.data.size() - value.data.size();
-                value.data = request.data;
+                value.setData(std::move(request.data));
             });
 
             container.updateValue(parentPath(request.path), [] (KeeperStorage::Node & parent)
@@ -675,9 +692,10 @@ struct KeeperStorageListRequestProcessor final : public KeeperStorageRequestProc
             if (path_prefix.empty())
                 throw DB::Exception("Logical error: path cannot be empty", ErrorCodes::LOGICAL_ERROR);
 
-            response.names.reserve(it->value.children.size());
+            const auto & children = it->value.getChildren();
+            response.names.reserve(children.size());
 
-            for (const auto child : it->value.children)
+            for (const auto child : children)
                 response.names.push_back(child.toString());
 
             response.stat = it->value.stat;
@@ -856,24 +874,23 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro
         for (const auto & sub_request : request.requests)
         {
             auto sub_zk_request = std::dynamic_pointer_cast<Coordination::ZooKeeperRequest>(sub_request);
-            if (sub_zk_request->getOpNum() == Coordination::OpNum::Create)
+            switch (sub_zk_request->getOpNum())
             {
-                concrete_requests.push_back(std::make_shared<KeeperStorageCreateRequestProcessor>(sub_zk_request));
+                case Coordination::OpNum::Create:
+                    concrete_requests.push_back(std::make_shared<KeeperStorageCreateRequestProcessor>(sub_zk_request));
+                    break;
+                case Coordination::OpNum::Remove:
+                    concrete_requests.push_back(std::make_shared<KeeperStorageRemoveRequestProcessor>(sub_zk_request));
+                    break;
+                case Coordination::OpNum::Set:
+                    concrete_requests.push_back(std::make_shared<KeeperStorageSetRequestProcessor>(sub_zk_request));
+                    break;
+                case Coordination::OpNum::Check:
+                    concrete_requests.push_back(std::make_shared<KeeperStorageCheckRequestProcessor>(sub_zk_request));
+                    break;
+                default:
+                    throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Illegal command as part of multi ZooKeeper request {}", sub_zk_request->getOpNum());
             }
-            else if (sub_zk_request->getOpNum() == Coordination::OpNum::Remove)
-            {
-                concrete_requests.push_back(std::make_shared<KeeperStorageRemoveRequestProcessor>(sub_zk_request));
-            }
-            else if (sub_zk_request->getOpNum() == Coordination::OpNum::Set)
-            {
-                concrete_requests.push_back(std::make_shared<KeeperStorageSetRequestProcessor>(sub_zk_request));
-            }
-            else if (sub_zk_request->getOpNum() == Coordination::OpNum::Check)
-            {
-                concrete_requests.push_back(std::make_shared<KeeperStorageCheckRequestProcessor>(sub_zk_request));
-            }
-            else
-                throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Illegal command as part of multi ZooKeeper request {}", sub_zk_request->getOpNum());
         }
     }
 
@@ -1092,8 +1109,7 @@ KeeperStorage::ResponsesForSessions KeeperStorage::processRequest(const Coordina
                     --parent.stat.numChildren;
                     ++parent.stat.cversion;
                     auto base_name = getBaseName(ephemeral_path);
-                    parent.children.erase(base_name);
-                    parent.size_bytes -= base_name.size;
+                    parent.removeChild(base_name);
                 });
 
                 container.erase(ephemeral_path);

src/Coordination/KeeperStorage.h

@@ -32,28 +32,38 @@ public:
 
     struct Node
     {
-        String data;
         uint64_t acl_id = 0; /// 0 -- no ACL by default
         bool is_sequental = false;
         Coordination::Stat stat{};
         int32_t seq_num = 0;
-        ChildrenSet children{};
         uint64_t size_bytes; // save size to avoid calculate every time
 
-        Node()
-        {
-            size_bytes = sizeof(size_bytes);
-            size_bytes += data.size();
-            size_bytes += sizeof(acl_id);
-            size_bytes += sizeof(is_sequental);
-            size_bytes += sizeof(stat);
-            size_bytes += sizeof(seq_num);
-        }
+        Node() : size_bytes(sizeof(Node)) { }
+
         /// Object memory size
         uint64_t sizeInBytes() const
         {
             return size_bytes;
         }
+
+        void setData(String new_data);
+
+        const auto & getData() const noexcept
+        {
+            return data;
+        }
+
+        void addChild(StringRef child_path);
+
+        void removeChild(StringRef child_path);
+
+        const auto & getChildren() const noexcept
+        {
+            return children;
+        }
+    private:
+        String data;
+        ChildrenSet children{};
     };
 
     struct ResponseForSession
@@ -104,7 +114,7 @@ public:
 
     /// Mapping session_id -> set of ephemeral nodes paths
     Ephemerals ephemerals;
-    /// Mapping sessuib_id -> set of watched nodes paths
+    /// Mapping session_id -> set of watched nodes paths
     SessionAndWatcher sessions_and_watchers;
     /// Expiration queue for session, allows to get dead sessions at some point of time
     SessionExpiryQueue session_expiry_queue;

src/Coordination/SnapshotableHashTable.h

@@ -80,7 +80,7 @@ private:
                     approximate_data_size += value_size;
                     if (!snapshot_mode)
                     {
-                        approximate_data_size += key_size;
+                        approximate_data_size -= key_size;
                         approximate_data_size -= old_value_size;
                     }
                 }
@@ -132,7 +132,6 @@ public:
 
         if (!it)
         {
-
             ListElem elem{copyStringInArena(arena, key), value, current_version};
             auto itr = list.insert(list.end(), std::move(elem));
             bool inserted;
@@ -228,7 +227,7 @@ public:
             /// We in snapshot mode but updating some node which is already more
             /// fresh than snapshot distance. So it will not participate in
             /// snapshot and we don't need to copy it.
-            if (snapshot_mode && list_itr->version <= snapshot_up_to_version)
+            if (list_itr->version <= snapshot_up_to_version)
             {
                 auto elem_copy = *(list_itr);
                 list_itr->active_in_map = false;

src/Coordination/ZooKeeperDataReader.cpp

@@ -98,7 +98,9 @@ int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in, Poco::L
     while (path != "/")
     {
         KeeperStorage::Node node{};
-        Coordination::read(node.data, in);
+        String data;
+        Coordination::read(data, in);
+        node.setData(std::move(data));
         Coordination::read(node.acl_id, in);
 
         /// Deserialize stat
@@ -117,7 +119,7 @@ int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in, Poco::L
         Coordination::read(node.stat.pzxid, in);
         if (!path.empty())
         {
-            node.stat.dataLength = node.data.length();
+            node.stat.dataLength = node.getData().length();
             node.seq_num = node.stat.cversion;
             storage.container.insertOrReplace(path, node);
 
@@ -137,7 +139,7 @@ int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in, Poco::L
         if (itr.key != "/")
         {
             auto parent_path = parentPath(itr.key);
-            storage.container.updateValue(parent_path, [path = itr.key] (KeeperStorage::Node & value) { value.children.insert(getBaseName(path)); value.stat.numChildren++; });
+            storage.container.updateValue(parent_path, [path = itr.key] (KeeperStorage::Node & value) { value.addChild(getBaseName(path)); value.stat.numChildren++; });
         }
     }
 

src/Coordination/tests/gtest_coordination.cpp

@@ -946,6 +946,8 @@ TEST_P(CoordinationTest, SnapshotableHashMapDataSize)
     EXPECT_EQ(hello.getApproximateDataSize(), 9);
     hello.updateValue("hello", [](IntNode & value) { value = 2; });
     EXPECT_EQ(hello.getApproximateDataSize(), 9);
+    hello.insertOrReplace("hello", 3);
+    EXPECT_EQ(hello.getApproximateDataSize(), 9);
 
     hello.erase("hello");
     EXPECT_EQ(hello.getApproximateDataSize(), 0);
@@ -958,6 +960,8 @@ TEST_P(CoordinationTest, SnapshotableHashMapDataSize)
     EXPECT_EQ(hello.getApproximateDataSize(), 9);
     hello.updateValue("hello", [](IntNode & value) { value = 2; });
     EXPECT_EQ(hello.getApproximateDataSize(), 18);
+    hello.insertOrReplace("hello", 1);
+    EXPECT_EQ(hello.getApproximateDataSize(), 27);
 
     hello.clearOutdatedNodes();
     EXPECT_EQ(hello.getApproximateDataSize(), 9);
@@ -972,31 +976,31 @@ TEST_P(CoordinationTest, SnapshotableHashMapDataSize)
     using Node = DB::KeeperStorage::Node;
     DB::SnapshotableHashTable<Node> world;
     Node n1;
-    n1.data = "1234";
+    n1.setData("1234");
     Node n2;
-    n2.data = "123456";
-    n2.children.insert("");
+    n2.setData("123456");
+    n2.addChild("");
 
     world.disableSnapshotMode();
     world.insert("world", n1);
-    EXPECT_EQ(world.getApproximateDataSize(), 98);
+    EXPECT_EQ(world.getApproximateDataSize(), 177);
     world.updateValue("world", [&](Node & value) { value = n2; });
-    EXPECT_EQ(world.getApproximateDataSize(), 98);
+    EXPECT_EQ(world.getApproximateDataSize(), 195);
 
     world.erase("world");
     EXPECT_EQ(world.getApproximateDataSize(), 0);
 
     world.enableSnapshotMode(100000);
     world.insert("world", n1);
-    EXPECT_EQ(world.getApproximateDataSize(), 98);
+    EXPECT_EQ(world.getApproximateDataSize(), 177);
     world.updateValue("world", [&](Node & value) { value = n2; });
-    EXPECT_EQ(world.getApproximateDataSize(), 196);
+    EXPECT_EQ(world.getApproximateDataSize(), 372);
 
     world.clearOutdatedNodes();
-    EXPECT_EQ(world.getApproximateDataSize(), 98);
+    EXPECT_EQ(world.getApproximateDataSize(), 195);
 
     world.erase("world");
-    EXPECT_EQ(world.getApproximateDataSize(), 98);
+    EXPECT_EQ(world.getApproximateDataSize(), 195);
 
     world.clear();
     EXPECT_EQ(world.getApproximateDataSize(), 0);
@@ -1006,7 +1010,7 @@ void addNode(DB::KeeperStorage & storage, const std::string & path, const std::s
 {
     using Node = DB::KeeperStorage::Node;
     Node node{};
-    node.data = data;
+    node.setData(data);
     node.stat.ephemeralOwner = ephemeral_owner;
     storage.container.insertOrReplace(path, node);
 }
@@ -1044,13 +1048,13 @@ TEST_P(CoordinationTest, TestStorageSnapshotSimple)
     auto [restored_storage, snapshot_meta, _] = manager.deserializeSnapshotFromBuffer(debuf);
 
     EXPECT_EQ(restored_storage->container.size(), 3);
-    EXPECT_EQ(restored_storage->container.getValue("/").children.size(), 1);
-    EXPECT_EQ(restored_storage->container.getValue("/hello").children.size(), 1);
-    EXPECT_EQ(restored_storage->container.getValue("/hello/somepath").children.size(), 0);
+    EXPECT_EQ(restored_storage->container.getValue("/").getChildren().size(), 1);
+    EXPECT_EQ(restored_storage->container.getValue("/hello").getChildren().size(), 1);
+    EXPECT_EQ(restored_storage->container.getValue("/hello/somepath").getChildren().size(), 0);
 
-    EXPECT_EQ(restored_storage->container.getValue("/").data, "");
-    EXPECT_EQ(restored_storage->container.getValue("/hello").data, "world");
-    EXPECT_EQ(restored_storage->container.getValue("/hello/somepath").data, "somedata");
+    EXPECT_EQ(restored_storage->container.getValue("/").getData(), "");
+    EXPECT_EQ(restored_storage->container.getValue("/hello").getData(), "world");
+    EXPECT_EQ(restored_storage->container.getValue("/hello/somepath").getData(), "somedata");
     EXPECT_EQ(restored_storage->session_id_counter, 7);
     EXPECT_EQ(restored_storage->zxid, 2);
     EXPECT_EQ(restored_storage->ephemerals.size(), 2);
@@ -1095,7 +1099,7 @@ TEST_P(CoordinationTest, TestStorageSnapshotMoreWrites)
     EXPECT_EQ(restored_storage->container.size(), 51);
     for (size_t i = 0; i < 50; ++i)
     {
-        EXPECT_EQ(restored_storage->container.getValue("/hello_" + std::to_string(i)).data, "world_" + std::to_string(i));
+        EXPECT_EQ(restored_storage->container.getValue("/hello_" + std::to_string(i)).getData(), "world_" + std::to_string(i));
     }
 }
 
@@ -1135,7 +1139,7 @@ TEST_P(CoordinationTest, TestStorageSnapshotManySnapshots)
 
     for (size_t i = 0; i < 250; ++i)
     {
-        EXPECT_EQ(restored_storage->container.getValue("/hello_" + std::to_string(i)).data, "world_" + std::to_string(i));
+        EXPECT_EQ(restored_storage->container.getValue("/hello_" + std::to_string(i)).getData(), "world_" + std::to_string(i));
     }
 }
 
@@ -1158,7 +1162,7 @@ TEST_P(CoordinationTest, TestStorageSnapshotMode)
         }
         for (size_t i = 0; i < 50; ++i)
         {
-            EXPECT_EQ(storage.container.getValue("/hello_" + std::to_string(i)).data, "wlrd_" + std::to_string(i));
+            EXPECT_EQ(storage.container.getValue("/hello_" + std::to_string(i)).getData(), "wlrd_" + std::to_string(i));
         }
         for (size_t i = 0; i < 50; ++i)
         {
@@ -1178,7 +1182,7 @@ TEST_P(CoordinationTest, TestStorageSnapshotMode)
     for (size_t i = 0; i < 50; ++i)
     {
         if (i % 2 != 0)
-            EXPECT_EQ(storage.container.getValue("/hello_" + std::to_string(i)).data, "wlrd_" + std::to_string(i));
+            EXPECT_EQ(storage.container.getValue("/hello_" + std::to_string(i)).getData(), "wlrd_" + std::to_string(i));
         else
             EXPECT_FALSE(storage.container.contains("/hello_" + std::to_string(i)));
     }
@@ -1187,7 +1191,7 @@ TEST_P(CoordinationTest, TestStorageSnapshotMode)
 
     for (size_t i = 0; i < 50; ++i)
     {
-        EXPECT_EQ(restored_storage->container.getValue("/hello_" + std::to_string(i)).data, "world_" + std::to_string(i));
+        EXPECT_EQ(restored_storage->container.getValue("/hello_" + std::to_string(i)).getData(), "world_" + std::to_string(i));
     }
 
 }
@@ -1310,7 +1314,7 @@ void testLogAndStateMachine(Coordination::CoordinationSettingsPtr settings, uint
     for (size_t i = 1; i < total_logs + 1; ++i)
     {
         auto path = "/hello_" + std::to_string(i);
-        EXPECT_EQ(source_storage.container.getValue(path).data, restored_storage.container.getValue(path).data);
+        EXPECT_EQ(source_storage.container.getValue(path).getData(), restored_storage.container.getValue(path).getData());
     }
 }
 
@@ -1585,13 +1589,13 @@ TEST_P(CoordinationTest, TestStorageSnapshotDifferentCompressions)
     auto [restored_storage, snapshot_meta, _] = new_manager.deserializeSnapshotFromBuffer(debuf);
 
     EXPECT_EQ(restored_storage->container.size(), 3);
-    EXPECT_EQ(restored_storage->container.getValue("/").children.size(), 1);
-    EXPECT_EQ(restored_storage->container.getValue("/hello").children.size(), 1);
-    EXPECT_EQ(restored_storage->container.getValue("/hello/somepath").children.size(), 0);
+    EXPECT_EQ(restored_storage->container.getValue("/").getChildren().size(), 1);
+    EXPECT_EQ(restored_storage->container.getValue("/hello").getChildren().size(), 1);
+    EXPECT_EQ(restored_storage->container.getValue("/hello/somepath").getChildren().size(), 0);
 
-    EXPECT_EQ(restored_storage->container.getValue("/").data, "");
-    EXPECT_EQ(restored_storage->container.getValue("/hello").data, "world");
-    EXPECT_EQ(restored_storage->container.getValue("/hello/somepath").data, "somedata");
+    EXPECT_EQ(restored_storage->container.getValue("/").getData(), "");
+    EXPECT_EQ(restored_storage->container.getValue("/hello").getData(), "world");
+    EXPECT_EQ(restored_storage->container.getValue("/hello/somepath").getData(), "somedata");
     EXPECT_EQ(restored_storage->session_id_counter, 7);
     EXPECT_EQ(restored_storage->zxid, 2);
     EXPECT_EQ(restored_storage->ephemerals.size(), 2);

src/Core/InterpolateDescription.cpp

@@ -0,0 +1,32 @@
+#include <Core/Block.h>
+#include <IO/Operators.h>
+#include <Common/JSONBuilder.h>
+#include <Core/InterpolateDescription.h>
+#include <Interpreters/convertFieldToType.h>
+
+namespace DB
+{
+
+    InterpolateDescription::InterpolateDescription(ActionsDAGPtr actions_, const Aliases & aliases)
+        : actions(actions_)
+    {
+        for (const auto & name_type : actions->getRequiredColumns())
+        {
+            if (const auto & p = aliases.find(name_type.name); p != aliases.end())
+                required_columns_map[p->second->getColumnName()] = name_type;
+            else
+                required_columns_map[name_type.name] = name_type;
+        }
+
+        for (const ColumnWithTypeAndName & column : actions->getResultColumns())
+        {
+            std::string name = column.name;
+            if (const auto & p = aliases.find(name); p != aliases.end())
+                name = p->second->getColumnName();
+
+            result_columns_set.insert(name);
+            result_columns_order.push_back(name);
+        }
+    }
+
+}

src/Core/InterpolateDescription.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <unordered_map>
+#include <memory>
+#include <cstddef>
+#include <string>
+#include <Core/Field.h>
+#include <Core/SettingsEnums.h>
+#include <Common/IntervalKind.h>
+#include <Parsers/ASTOrderByElement.h>
+#include <Parsers/ASTInterpolateElement.h>
+#include <Functions/FunctionsMiscellaneous.h>
+#include <Interpreters/Aliases.h>
+
+
+namespace DB
+{
+
+/// Interpolate description
+struct InterpolateDescription
+{
+    explicit InterpolateDescription(ActionsDAGPtr actions, const Aliases & aliases);
+
+    ActionsDAGPtr actions;
+
+    std::unordered_map<std::string, NameAndTypePair> required_columns_map; /// input column name -> {alias, type}
+    std::unordered_set<std::string> result_columns_set; /// result block columns
+    std::vector<std::string> result_columns_order; /// result block columns order
+};
+
+using InterpolateDescriptionPtr = std::shared_ptr<InterpolateDescription>;
+
+}

src/Core/Settings.h

@@ -199,6 +199,7 @@ static constexpr UInt64 operator""_Gb(unsigned long long value)
     M(UInt64, log_queries_cut_to_length, 100000, "If query length is greater than specified threshold (in bytes), then cut query when writing to query log. Also limit length of printed query in ordinary text log.", 0) \
     M(Float, log_queries_probability, 1., "Log queries with the specified probabality.", 0) \
     \
+    M(Bool, log_processors_profiles, false, "Log Processors profile events.", 0) \
     M(DistributedProductMode, distributed_product_mode, DistributedProductMode::DENY, "How are distributed subqueries performed inside IN or JOIN sections?", IMPORTANT) \
     \
     M(UInt64, max_concurrent_queries_for_all_users, 0, "The maximum number of concurrent requests for all users.", 0) \
@@ -468,6 +469,7 @@ static constexpr UInt64 operator""_Gb(unsigned long long value)
     M(Bool, use_compact_format_in_distributed_parts_names, true, "Changes format of directories names for distributed table insert parts.", 0) \
     M(Bool, validate_polygons, true, "Throw exception if polygon is invalid in function pointInPolygon (e.g. self-tangent, self-intersecting). If the setting is false, the function will accept invalid polygons but may silently return wrong result.", 0) \
     M(UInt64, max_parser_depth, DBMS_DEFAULT_MAX_PARSER_DEPTH, "Maximum parser depth (recursion depth of recursive descend parser).", 0) \
+    M(Bool, allow_settings_after_format_in_insert, false, "Allow SETTINGS after FORMAT, but note, that this is not always safe (note: this is a compatibility setting).", 0) \
     M(Seconds, temporary_live_view_timeout, DEFAULT_TEMPORARY_LIVE_VIEW_TIMEOUT_SEC, "Timeout after which temporary live view is deleted.", 0) \
     M(Seconds, periodic_live_view_refresh, DEFAULT_PERIODIC_LIVE_VIEW_REFRESH_SEC, "Interval after which periodically refreshed live view is forced to refresh.", 0) \
     M(Bool, transform_null_in, false, "If enabled, NULL values will be matched with 'IN' operator as if they are considered equal.", 0) \
@@ -563,8 +565,10 @@ static constexpr UInt64 operator""_Gb(unsigned long long value)
     \
     M(UInt64, remote_fs_read_max_backoff_ms, 10000, "Max wait time when trying to read data for remote disk", 0) \
     M(UInt64, remote_fs_read_backoff_max_tries, 5, "Max attempts to read with backoff", 0) \
-    M(Bool, remote_fs_enable_cache, true, "Use cache for remote filesystem. This setting does not turn on/off cache for disks (must me done via disk config), but allows to bypass cache for some queries if intended", 0) \
-    M(UInt64, remote_fs_cache_max_wait_sec, 5, "Allow to wait at most this number of seconds for download of current remote_fs_buffer_size bytes, and skip cache if exceeded", 0) \
+    M(Bool, enable_filesystem_cache, true, "Use cache for remote filesystem. This setting does not turn on/off cache for disks (must me done via disk config), but allows to bypass cache for some queries if intended", 0) \
+    M(UInt64, filesystem_cache_max_wait_sec, 5, "Allow to wait at most this number of seconds for download of current remote_fs_buffer_size bytes, and skip cache if exceeded", 0) \
+    M(Bool, enable_filesystem_cache_on_write_operations, false, "Write into cache on write operations. To actually work this setting requires be added to disk config too", 0) \
+    M(Bool, read_from_filesystem_cache_if_exists_otherwise_bypass_cache, false, "", 0) \
     \
     M(UInt64, http_max_tries, 10, "Max attempts to read via http.", 0) \
     M(UInt64, http_retry_initial_backoff_ms, 100, "Min milliseconds for backoff, when retrying read via http", 0) \
@@ -581,6 +585,7 @@ static constexpr UInt64 operator""_Gb(unsigned long long value)
     M(Bool, allow_experimental_nlp_functions, false, "Enable experimental functions for natural language processing.", 0) \
     M(Bool, allow_experimental_object_type, false, "Allow Object and JSON data types", 0) \
     M(String, insert_deduplication_token, "", "If not empty, used for duplicate detection instead of data digest", 0) \
+    M(Bool, throw_on_unsupported_query_inside_transaction, true, "Throw exception if unsupported query is used inside transaction", 0) \
 // End of COMMON_SETTINGS
 // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS.
 
@@ -641,6 +646,13 @@ static constexpr UInt64 operator""_Gb(unsigned long long value)
     M(UInt64, input_format_msgpack_number_of_columns, 0, "The number of columns in inserted MsgPack data. Used for automatic schema inference from data.", 0) \
     M(MsgPackUUIDRepresentation, output_format_msgpack_uuid_representation, FormatSettings::MsgPackUUIDRepresentation::EXT, "The way how to output UUID in MsgPack format.", 0) \
     M(UInt64, input_format_max_rows_to_read_for_schema_inference, 100, "The maximum rows of data to read for automatic schema inference", 0) \
+    M(Bool, input_format_csv_use_best_effort_in_schema_inference, true, "Use some tweaks and heuristics to infer schema in CSV format", 0) \
+    M(Bool, input_format_tsv_use_best_effort_in_schema_inference, true, "Use some tweaks and heuristics to infer schema in TSV format", 0) \
+    M(Bool, input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference, false, "Allow to skip columns with unsupported types while schema inference for format Parquet", 0) \
+    M(Bool, input_format_orc_skip_columns_with_unsupported_types_in_schema_inference, false, "Allow to skip columns with unsupported types while schema inference for format ORC", 0) \
+    M(Bool, input_format_arrow_skip_columns_with_unsupported_types_in_schema_inference, false, "Allow to skip columns with unsupported types while schema inference for format Arrow", 0) \
+    M(String, column_names_for_schema_inference, "", "The list of column names to use in schema inference for formats without column names. The format: 'column1,column2,column3,...'", 0) \
+    M(Bool, input_format_json_read_bools_as_numbers, true, "Allow to parse bools as numbers in JSON input formats", 0) \
     \
     M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic', 'best_effort' and 'best_effort_us'.", 0) \
     M(DateTimeOutputFormat, date_time_output_format, FormatSettings::DateTimeOutputFormat::Simple, "Method to write DateTime to text output. Possible values: 'simple', 'iso', 'unix_timestamp'.", 0) \

src/Core/SettingsEnums.h

@@ -165,7 +165,7 @@ DECLARE_SETTING_ENUM(DistributedDDLOutputMode)
 
 enum class HandleKafkaErrorMode
 {
-    DEFAULT = 0, // Ignore errors whit threshold.
+    DEFAULT = 0, // Ignore errors with threshold.
     STREAM, // Put errors to stream in the virtual column named ``_error.
     /*FIXED_SYSTEM_TABLE, Put errors to in a fixed system table likey system.kafka_errors. This is not implemented now.  */
     /*CUSTOM_SYSTEM_TABLE, Put errors to in a custom system table. This is not implemented now.  */

src/Core/SortDescription.h

@@ -7,6 +7,7 @@
 #include <Core/Field.h>
 #include <Core/SettingsEnums.h>
 #include <Common/IntervalKind.h>
+#include <DataTypes/IDataType.h>
 
 class Collator;
 

src/DataTypes/DataTypeLowCardinalityHelpers.cpp

@@ -36,7 +36,7 @@ DataTypePtr recursiveRemoveLowCardinality(const DataTypePtr & type)
             element = recursiveRemoveLowCardinality(element);
 
         if (tuple_type->haveExplicitNames())
-            return std::make_shared<DataTypeTuple>(elements, tuple_type->getElementNames(), tuple_type->serializeNames());
+            return std::make_shared<DataTypeTuple>(elements, tuple_type->getElementNames());
         else
             return std::make_shared<DataTypeTuple>(elements);
     }

src/DataTypes/DataTypeMap.cpp

@@ -45,22 +45,7 @@ DataTypeMap::DataTypeMap(const DataTypePtr & key_type_, const DataTypePtr & valu
 
 void DataTypeMap::assertKeyType() const
 {
-    bool type_error = false;
-    if (key_type->getTypeId() == TypeIndex::LowCardinality)
-    {
-        const auto & low_cardinality_data_type = assert_cast<const DataTypeLowCardinality &>(*key_type);
-        if (!isStringOrFixedString(*(low_cardinality_data_type.getDictionaryType())))
-            type_error = true;
-    }
-    else if (!key_type->isValueRepresentedByInteger()
-        && !isStringOrFixedString(*key_type)
-        && !WhichDataType(key_type).isNothing()
-        && !WhichDataType(key_type).isUUID())
-    {
-        type_error = true;
-    }
-
-    if (type_error)
+    if (!checkKeyType(key_type))
         throw Exception(ErrorCodes::BAD_ARGUMENTS,
             "Type of Map key must be a type, that can be represented by integer or String or FixedString (possibly LowCardinality) or UUID,"
             " but {} given", key_type->getName());
@@ -102,6 +87,25 @@ bool DataTypeMap::equals(const IDataType & rhs) const
     return nested->equals(*rhs_map.nested);
 }
 
+bool DataTypeMap::checkKeyType(DataTypePtr key_type)
+{
+    if (key_type->getTypeId() == TypeIndex::LowCardinality)
+    {
+        const auto & low_cardinality_data_type = assert_cast<const DataTypeLowCardinality &>(*key_type);
+        if (!isStringOrFixedString(*(low_cardinality_data_type.getDictionaryType())))
+            return false;
+    }
+    else if (!key_type->isValueRepresentedByInteger()
+             && !isStringOrFixedString(*key_type)
+             && !WhichDataType(key_type).isNothing()
+             && !WhichDataType(key_type).isUUID())
+    {
+        return false;
+    }
+
+    return true;
+}
+
 static DataTypePtr create(const ASTPtr & arguments)
 {
     if (!arguments || arguments->children.size() != 2)

src/DataTypes/DataTypeMap.h

@@ -48,6 +48,8 @@ public:
 
     SerializationPtr doGetDefaultSerialization() const override;
 
+    static bool checkKeyType(DataTypePtr key_type);
+
 private:
     void assertKeyType() const;
 };

src/DataTypes/DataTypeTuple.cpp

@@ -64,8 +64,8 @@ static std::optional<Exception> checkTupleNames(const Strings & names)
     return {};
 }
 
-DataTypeTuple::DataTypeTuple(const DataTypes & elems_, const Strings & names_, bool serialize_names_)
-    : elems(elems_), names(names_), have_explicit_names(true), serialize_names(serialize_names_)
+DataTypeTuple::DataTypeTuple(const DataTypes & elems_, const Strings & names_)
+    : elems(elems_), names(names_), have_explicit_names(true)
 {
     size_t size = elems.size();
     if (names.size() != size)
@@ -75,11 +75,6 @@ DataTypeTuple::DataTypeTuple(const DataTypes & elems_, const Strings & names_, b
         throw std::move(*exception);
 }
 
-bool DataTypeTuple::canBeCreatedWithNames(const Strings & names)
-{
-    return checkTupleNames(names) == std::nullopt;
-}
-
 std::string DataTypeTuple::doGetName() const
 {
     size_t size = elems.size();
@@ -91,7 +86,7 @@ std::string DataTypeTuple::doGetName() const
         if (i != 0)
             s << ", ";
 
-        if (have_explicit_names && serialize_names)
+        if (have_explicit_names)
             s << backQuoteIfNeed(names[i]) << ' ';
 
         s << elems[i]->getName();
@@ -206,7 +201,7 @@ bool DataTypeTuple::equals(const IDataType & rhs) const
         return false;
 
     for (size_t i = 0; i < size; ++i)
-        if (!elems[i]->equals(*rhs_tuple.elems[i]))
+        if (!elems[i]->equals(*rhs_tuple.elems[i]) || names[i] != rhs_tuple.names[i])
             return false;
 
     return true;
@@ -265,31 +260,29 @@ size_t DataTypeTuple::getSizeOfValueInMemory() const
 SerializationPtr DataTypeTuple::doGetDefaultSerialization() const
 {
     SerializationTuple::ElementSerializations serializations(elems.size());
-    bool use_explicit_names = have_explicit_names && serialize_names;
     for (size_t i = 0; i < elems.size(); ++i)
     {
-        String elem_name = use_explicit_names ? names[i] : toString(i + 1);
+        String elem_name = have_explicit_names ? names[i] : toString(i + 1);
         auto serialization = elems[i]->getDefaultSerialization();
         serializations[i] = std::make_shared<SerializationNamed>(serialization, elem_name);
     }
 
-    return std::make_shared<SerializationTuple>(std::move(serializations), use_explicit_names);
+    return std::make_shared<SerializationTuple>(std::move(serializations), have_explicit_names);
 }
 
 SerializationPtr DataTypeTuple::getSerialization(const SerializationInfo & info) const
 {
     SerializationTuple::ElementSerializations serializations(elems.size());
     const auto & info_tuple = assert_cast<const SerializationInfoTuple &>(info);
-    bool use_explicit_names = have_explicit_names && serialize_names;
 
     for (size_t i = 0; i < elems.size(); ++i)
     {
-        String elem_name = use_explicit_names ? names[i] : toString(i + 1);
+        String elem_name = have_explicit_names ? names[i] : toString(i + 1);
         auto serialization = elems[i]->getSerialization(*info_tuple.getElementInfo(i));
         serializations[i] = std::make_shared<SerializationNamed>(serialization, elem_name);
     }
 
-    return std::make_shared<SerializationTuple>(std::move(serializations), use_explicit_names);
+    return std::make_shared<SerializationTuple>(std::move(serializations), have_explicit_names);
 }
 
 MutableSerializationInfoPtr DataTypeTuple::createSerializationInfo(const SerializationInfo::Settings & settings) const

src/DataTypes/DataTypeTuple.h

@@ -22,14 +22,11 @@ private:
     DataTypes elems;
     Strings names;
     bool have_explicit_names;
-    bool serialize_names = true;
 public:
     static constexpr bool is_parametric = true;
 
     explicit DataTypeTuple(const DataTypes & elems);
-    DataTypeTuple(const DataTypes & elems, const Strings & names, bool serialize_names_ = true);
-
-    static bool canBeCreatedWithNames(const Strings & names);
+    DataTypeTuple(const DataTypes & elems, const Strings & names);
 
     TypeIndex getTypeId() const override { return TypeIndex::Tuple; }
     std::string doGetName() const override;
@@ -66,7 +63,6 @@ public:
     String getNameByPosition(size_t i) const;
 
     bool haveExplicitNames() const { return have_explicit_names; }
-    bool serializeNames() const { return serialize_names; }
 };
 
 }

src/DataTypes/Serializations/SerializationNumber.cpp

@@ -43,7 +43,7 @@ void SerializationNumber<T>::serializeTextJSON(const IColumn & column, size_t ro
 }
 
 template <typename T>
-void SerializationNumber<T>::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
+void SerializationNumber<T>::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     bool has_quote = false;
     if (!istr.eof() && *istr.position() == '"')        /// We understand the number both in quotes and without.
@@ -67,7 +67,7 @@ void SerializationNumber<T>::deserializeTextJSON(IColumn & column, ReadBuffer &
         static constexpr bool is_uint8 = std::is_same_v<T, UInt8>;
         static constexpr bool is_int8 = std::is_same_v<T, Int8>;
 
-        if (is_uint8 || is_int8)
+        if (settings.json.read_bools_as_numbers || is_uint8 || is_int8)
         {
             // extra conditions to parse true/false strings into 1/0
             if (istr.eof())