autorelease to check builds and functional tests

.github/actions/clean/action.yml

@@ -1,11 +1,23 @@
 name: Clean runner
 description: Clean the runner's temp path on ending
+inputs:
+  images:
+    description: clean docker images
+    default: false
+    type: boolean
 runs:
   using: "composite"
   steps:
-    - name: Clean
+    - name: Clean Temp
       shell: bash
       run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "${{runner.temp}}"
+        sudo rm -fr "${{runner.temp}}"
+    - name: Clean Docker Containers
+      shell: bash
+      run: |
+          docker rm -vf $(docker ps -aq) ||:
+    - name: Clean Docker Images
+      if: ${{ inputs.images }}
+      shell: bash
+      run: |
+        docker rmi -f $(docker images -aq) ||:

.github/workflows/auto_releases.yml

@@ -14,7 +14,7 @@ on:
       dry-run:
         description: 'Dry run'
         required: false
-        default: true
+        default: false
         type: boolean
 
 jobs:
@@ -51,7 +51,11 @@ jobs:
               cat  /tmp/autorelease_params.json
               echo 'EOF'
           } >> "$GITHUB_OUTPUT"
-          echo "DRY_RUN=true" >> "$GITHUB_OUTPUT"
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            echo "DRY_RUN=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "DRY_RUN=${{ github.event.inputs.dry-run }}" >> "$GITHUB_OUTPUT"
+          fi
       - name: Post Release Branch statuses
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
@@ -74,14 +78,22 @@ jobs:
     secrets:
       ROBOT_CLICKHOUSE_COMMIT_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}
 
-  PostSlackMessage:
-    needs: [AutoReleaseInfo]
+  CleanUp:
+    needs: [Releases]
     runs-on: [self-hosted, release-maker]
-    if: ${{ !cancelled() }}
     steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-      - name: Post
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 auto_release.py --post-auto-release-complete --wf-status ${{ job.status }}
+      - uses: ./.github/actions/clean
+        with:
+          images: true
+
+#  PostSlackMessage:
+#    needs: [Releases]
+#    runs-on: [self-hosted, release-maker]
+#    if: ${{ !cancelled() }}
+#    steps:
+#      - name: Check out repository code
+#        uses: ClickHouse/checkout@v1
+#      - name: Post
+#        run: |
+#          cd "$GITHUB_WORKSPACE/tests/ci"
+#          python3 auto_release.py --post-auto-release-complete --wf-status ${{ job.status }}

tests/ci/artifactory.py

@@ -143,6 +143,8 @@ class DebianArtifactory:
             print(f"  {cmd}")
             Shell.check(cmd, strict=True)
             Shell.check("sync")
+        time.sleep(10)
+        Shell.check(f"lsof +D R2MountPoint.MOUNT_POINT", verbose=True)
 
     def test_packages(self):
         Shell.check("docker pull ubuntu:latest", strict=True)

tests/ci/auto_release.py

@@ -1,5 +1,4 @@
 import argparse
-import copy
 import dataclasses
 import json
 import os
@@ -77,8 +76,10 @@ class AutoReleaseInfo:
             print(json.dumps(dataclasses.asdict(self), indent=2), file=f)
 
         # dump file for GH action matrix that is similar to the file above but with dropped not ready release branches
-        params = copy.deepcopy(self)
-        params.releases = [release for release in params.releases if release.ready]
+        params = dataclasses.asdict(self)
+        params["releases"] = [
+            release for release in params["releases"] if release["ready"]
+        ]
         with open(AUTORELEASE_MATRIX_PARAMS, "w", encoding="utf-8") as f:
             print(json.dumps(params, indent=2), file=f)
 
@@ -110,7 +111,6 @@ def _prepare(token):
         refs = list(repo.get_git_matching_refs(f"tags/v{pr.head.ref}"))
         assert refs
 
-        refs.sort(key=lambda ref: ref.ref)
         latest_release_tag_ref = refs[-1]
         latest_release_tag = repo.get_git_tag(latest_release_tag_ref.object.sha)
 
@@ -118,6 +118,10 @@ def _prepare(token):
             f"git rev-list --first-parent {latest_release_tag.tag}..origin/{pr.head.ref}",
         ).split("\n")
         commit_num = len(commits)
+        if latest_release_tag.tag.endswith("new"):
+            print("It's a new release branch - skip auto release for it")
+            continue
+
         print(
             f"Previous release [{latest_release_tag.tag}] was [{commit_num}] commits ago, date [{latest_release_tag.tagger.date}]"
         )
@@ -141,17 +145,33 @@ def _prepare(token):
                 commits_to_branch_head += 1
                 continue
 
-            commit_ci_status = CI.GH.get_commit_status_by_name(
-                token=token,
-                commit_sha=commit,
-                # handle old name for old releases
-                status_name=(CI.JobNames.BUILD_CHECK, "ClickHouse build check"),
-            )
+            # TODO: switch to check if CI is entirely green
+            statuses = [
+                CI.GH.get_commit_status_by_name(
+                    token=token,
+                    commit_sha=commit,
+                    # handle old name for old releases
+                    status_name=(CI.JobNames.BUILD_CHECK, "ClickHouse build check"),
+                ),
+                CI.GH.get_commit_status_by_name(
+                    token=token,
+                    commit_sha=commit,
+                    # handle old name for old releases
+                    status_name=CI.JobNames.STATELESS_TEST_RELEASE,
+                ),
+                CI.GH.get_commit_status_by_name(
+                    token=token,
+                    commit_sha=commit,
+                    # handle old name for old releases
+                    status_name=CI.JobNames.STATEFUL_TEST_RELEASE,
+                ),
+            ]
             commit_sha = commit
-            if commit_ci_status == SUCCESS:
+            if any(status == SUCCESS for status in statuses):
+                commit_ci_status = SUCCESS
                 break
 
-            print(f"CI status [{commit_ci_status}] - skip")
+            print(f"CI status [{statuses}] - skip")
             commits_to_branch_head += 1
 
         ready = False

tests/ci/ci_utils.py

@@ -118,7 +118,7 @@ class GH:
                 statuses = response.json()
                 for status in statuses:
                     if status["context"] in status_name:
-                        return status["state"]
+                        return status["state"]  # type: ignore
 
                 # Check if there is a next page
                 url = response.links.get("next", {}).get("url")