Merge pull request #54857 from ClickHouse/speedup-images-enrich

Use `--filter` to reduce checkout time

.github/workflows/backport_branches.yml

@@ -77,6 +77,7 @@ jobs:
         with:
           clear-repository: true
           fetch-depth: 0  # to find ancestor merge commits necessary for finding proper docker tags
+          filter: tree:0
       - name: Download changed aarch64 images
         uses: actions/download-artifact@v3
         with:
@@ -185,6 +186,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # For a proper version and performance artifacts
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -227,6 +229,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # For a proper version and performance artifacts
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -399,6 +402,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
         run: |
           rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
@@ -448,6 +452,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
         run: |
           rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
@@ -487,6 +492,7 @@ jobs:
         with:
           clear-repository: true
           fetch-depth: 0  # It MUST BE THE SAME for all dependencies and the job itself
+          filter: tree:0
       - name: Check docker clickhouse/clickhouse-server building
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"

.github/workflows/docs_check.yml

@@ -18,6 +18,7 @@ on:  # yamllint disable-line rule:truthy
       - 'docs/**'
       - 'utils/check-style/aspell-ignore/**'
       - 'tests/ci/docs_check.py'
+      - '.github/workflows/docs_check.yml'
 jobs:
   CheckLabels:
     runs-on: [self-hosted, style-checker]
@@ -73,6 +74,7 @@ jobs:
         with:
           clear-repository: true
           fetch-depth: 0  # to find ancestor merge commits necessary for finding proper docker tags
+          filter: tree:0
       - name: Download changed aarch64 images
         uses: actions/download-artifact@v3
         with:

.github/workflows/jepsen.yml

@@ -24,6 +24,7 @@ jobs:
         with:
           clear-repository: true
           fetch-depth: 0
+          filter: tree:0
       - name: Jepsen Test
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -53,6 +54,7 @@ jobs:
   #       with:
   #         clear-repository: true
   #         fetch-depth: 0
+  #         filter: tree:0
   #     - name: Jepsen Test
   #       run: |
   #         sudo rm -fr "$TEMP_PATH"

.github/workflows/master.yml

@@ -61,6 +61,7 @@ jobs:
         with:
           clear-repository: true
           fetch-depth: 0  # to find ancestor merge commits necessary for finding proper docker tags
+          filter: tree:0
       - name: Download changed aarch64 images
         uses: actions/download-artifact@v3
         with:
@@ -200,6 +201,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # For a proper version and performance artifacts
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -242,6 +244,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # For a proper version and performance artifacts
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -283,6 +286,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -581,6 +585,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
         run: |
           rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
@@ -630,6 +635,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -672,6 +678,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -714,6 +721,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
         run: |
           rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
@@ -763,6 +771,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -805,6 +814,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -847,6 +857,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -889,6 +900,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -931,6 +943,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -963,6 +976,7 @@ jobs:
         with:
           clear-repository: true
           fetch-depth: 0  # It MUST BE THE SAME for all dependencies and the job itself
+          filter: tree:0
       - name: Check docker clickhouse/clickhouse-server building
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"

.github/workflows/nightly.yml

@@ -54,6 +54,7 @@ jobs:
         with:
           clear-repository: true
           fetch-depth: 0  # to find ancestor merge commits necessary for finding proper docker tags
+          filter: tree:0
       - name: Download changed aarch64 images
         uses: actions/download-artifact@v3
         with:
@@ -90,6 +91,7 @@ jobs:
         with:
           clear-repository: true
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+          filter: tree:0
           submodules: true
       - name: Set up JDK 11
         uses: actions/setup-java@v1

.github/workflows/pull_request.yml

@@ -18,6 +18,7 @@ on:  # yamllint disable-line rule:truthy
       - 'docs/**'
       - 'utils/check-style/aspell-ignore/**'
       - 'tests/ci/docs_check.py'
+      - '.github/workflows/docs_check.yml'
 ##########################################################################################
 ##################################### SMALL CHECKS #######################################
 ##########################################################################################
@@ -94,6 +95,7 @@ jobs:
         with:
           clear-repository: true
           fetch-depth: 0  # to find ancestor merge commits necessary for finding proper docker tags
+          filter: tree:0
       - name: Download changed aarch64 images
         uses: actions/download-artifact@v3
         with:
@@ -266,6 +268,7 @@ jobs:
         with:
           clear-repository: true
           fetch-depth: 0  # for performance artifact
+          filter: tree:0
           submodules: true
       - name: Build
         run: |
@@ -350,6 +353,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0  # for performance artifact
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -1021,6 +1025,7 @@ jobs:
         with:
           clear-repository: true
           fetch-depth: 0  # It MUST BE THE SAME for all dependencies and the job itself
+          filter: tree:0
       - name: Check docker clickhouse/clickhouse-server building
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"

.github/workflows/release.yml

@@ -49,6 +49,7 @@ jobs:
       with:
         clear-repository: true
         fetch-depth: 0  # otherwise we will have no version info
+        filter: tree:0
         ref: ${{ env.GITHUB_TAG }}
     - name: Check docker clickhouse/clickhouse-server building
       run: |

.github/workflows/release_branches.yml

@@ -53,6 +53,7 @@ jobs:
         with:
           clear-repository: true
           fetch-depth: 0  # to find ancestor merge commits necessary for finding proper docker tags
+          filter: tree:0
       - name: Download changed aarch64 images
         uses: actions/download-artifact@v3
         with:
@@ -161,6 +162,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -203,6 +205,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # For a proper version and performance artifacts
+          filter: tree:0
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"
@@ -456,6 +459,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
         run: |
           rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
@@ -505,6 +509,7 @@ jobs:
           clear-repository: true
           submodules: true
           fetch-depth: 0 # otherwise we will have no info about contributors
+          filter: tree:0
       - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
         run: |
           rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
@@ -544,6 +549,7 @@ jobs:
         with:
           clear-repository: true
           fetch-depth: 0  # It MUST BE THE SAME for all dependencies and the job itself
+          filter: tree:0
       - name: Check docker clickhouse/clickhouse-server building
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"

.github/workflows/tags_stable.yml

@@ -38,6 +38,7 @@ jobs:
       with:
         ref: master
         fetch-depth: 0
+        filter: tree:0
     - name: Update versions, docker version, changelog, security
       env:
         GITHUB_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}

docker/packager/binary/build.sh

@@ -172,10 +172,15 @@ then
     # This is why we add this repository snapshot from CI to the performance test
     # package.
     mkdir "$PERF_OUTPUT"/ch
-    git -C "$PERF_OUTPUT"/ch init --bare
-    git -C "$PERF_OUTPUT"/ch remote add origin /build
-    git -C "$PERF_OUTPUT"/ch fetch --no-tags --depth 50 origin HEAD:pr
-    git -C "$PERF_OUTPUT"/ch fetch --no-tags --depth 50 origin master:master
+    # Copy .git only, but skip modules, using tar
+    tar c -C /build/ --exclude='.git/modules/**' .git | tar x -C "$PERF_OUTPUT"/ch
+    # Create branch pr and origin/master to have them for the following performance comparison
+    git -C "$PERF_OUTPUT"/ch branch pr
+    git -C "$PERF_OUTPUT"/ch fetch --no-tags --depth 50 origin master:origin/master
+    # Clean remote, to not have it stale
+    git -C "$PERF_OUTPUT"/ch remote | xargs -n1 git -C "$PERF_OUTPUT"/ch remote remove
+    # And clean all tags
+    git -C "$PERF_OUTPUT"/ch tag | xargs git -C "$PERF_OUTPUT"/ch tag -d
     git -C "$PERF_OUTPUT"/ch reset --soft pr
     git -C "$PERF_OUTPUT"/ch log -5
     (