Merge pull request #68525 from ClickHouse/ci_autoreleases_continue

CI: Auto Releases in prod

.github/actions/clean/action.yml

@@ -1,11 +1,23 @@
 name: Clean runner
 description: Clean the runner's temp path on ending
+inputs:
+  images:
+    description: clean docker images
+    default: false
+    type: boolean
 runs:
   using: "composite"
   steps:
-    - name: Clean
+    - name: Clean Temp
       shell: bash
       run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "${{runner.temp}}"
+        sudo rm -fr "${{runner.temp}}"
+    - name: Clean Docker Containers
+      shell: bash
+      run: |
+          docker rm -vf $(docker ps -aq) ||:
+    - name: Clean Docker Images
+      if: ${{ inputs.images }}
+      shell: bash
+      run: |
+        docker rmi -f $(docker images -aq) ||:

.github/workflows/auto_releases.yml

@@ -14,18 +14,16 @@ on:
       dry-run:
         description: 'Dry run'
         required: false
-        default: true
+        default: false
         type: boolean
 
 jobs:
   AutoReleaseInfo:
-    runs-on: [self-hosted, style-checker-aarch64]
+    runs-on: [self-hosted, release-maker]
     outputs:
       data: ${{ steps.info.outputs.AUTO_RELEASE_PARAMS }}
       dry_run: ${{ steps.info.outputs.DRY_RUN }}
     steps:
-      - name: Debug Info
-        uses: ./.github/actions/debug
       - name: Set envs
         run: |
           cat >> "$GITHUB_ENV" << 'EOF'
@@ -36,6 +34,10 @@ jobs:
           echo "DRY_RUN=true" >> "$GITHUB_ENV"
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
+        with:
+          fetch-depth: 0  # full history needed
+      - name: Debug Info
+        uses: ./.github/actions/debug
       - name: Prepare Info
         id: info
         run: |
@@ -46,15 +48,14 @@ jobs:
           echo "::endgroup::"
           {
               echo 'AUTO_RELEASE_PARAMS<<EOF'
-              cat  /tmp/autorelease_info.json
-              echo 'EOF'
-          } >> "$GITHUB_ENV"
-          {
-              echo 'AUTO_RELEASE_PARAMS<<EOF'
-              cat  /tmp/autorelease_info.json
+              cat  /tmp/autorelease_params.json
               echo 'EOF'
           } >> "$GITHUB_OUTPUT"
-          echo "DRY_RUN=true" >> "$GITHUB_OUTPUT"
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            echo "DRY_RUN=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "DRY_RUN=${{ github.event.inputs.dry-run }}" >> "$GITHUB_OUTPUT"
+          fi
       - name: Post Release Branch statuses
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
@@ -62,48 +63,37 @@ jobs:
       - name: Clean up
         uses: ./.github/actions/clean
 
-  Release_0:
+  Releases:
     needs: AutoReleaseInfo
-    name: Release ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[0].release_branch }}
-    if: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[0] && fromJson(needs.AutoReleaseInfo.outputs.data).releases[0].ready }}
+    strategy:
+      matrix:
+        release_params: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases }}
+      max-parallel: 1
+    name: Release ${{ matrix.release_params.release_branch }}
     uses: ./.github/workflows/create_release.yml
     with:
-      ref: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[0].commit_sha }}
+      ref: ${{ matrix.release_params.commit_sha }}
       type: patch
-      dry-run: ${{ needs.AutoReleaseInfo.outputs.dry_run }}
-#
-#  Release_1:
-#    needs: [AutoReleaseInfo, Release_0]
-#    name: Release ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[1].release_branch }}
-#    if: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[1] && fromJson(needs.AutoReleaseInfo.outputs.data).releases[1].ready }}
-#    uses: ./.github/workflows/create_release.yml
-#    with:
-#      ref: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[1].commit_sha }}
-#      type: patch
-#      dry-run: ${{ env.DRY_RUN }}
-#
-#  Release_2:
-#    needs: [AutoReleaseInfo, Release_1]
-#    name: Release ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[2].release_branch }}
-#    if: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[0] && fromJson(needs.AutoReleaseInfo.outputs.data).releases[2].ready }}
-#    uses: ./.github/workflow/create_release.yml
-#    with:
-#      ref: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[0].commit_sha }}
-#      type: patch
-#      dry-run: ${{ env.DRY_RUN }}
-#
-#  Release_3:
-#    needs: [AutoReleaseInfo, Release_2]
-#    name: Release ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[3].release_branch }}
-#    if: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[3] && fromJson(needs.AutoReleaseInfo.outputs.data).releases[3].ready }}
-#    uses: ./.github/workflow/create_release.yml
-#    with:
-#      ref: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[3].commit_sha }}
-#      type: patch
-#      dry-run: ${{ env.DRY_RUN }}
+      dry-run: ${{ fromJson(needs.AutoReleaseInfo.outputs.dry_run) }}
+    secrets:
+      ROBOT_CLICKHOUSE_COMMIT_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}
 
-#  - name: Post Slack Message
+  CleanUp:
+    needs: [Releases]
+    runs-on: [self-hosted, release-maker]
+    steps:
+      - uses: ./.github/actions/clean
+        with:
+          images: true
+
+#  PostSlackMessage:
+#    needs: [Releases]
+#    runs-on: [self-hosted, release-maker]
 #    if: ${{ !cancelled() }}
-#    run: |
-#      cd "$GITHUB_WORKSPACE/tests/ci"
-#      python3 auto_release.py --post-auto-release-complete --wf-status ${{ job.status }}
+#    steps:
+#      - name: Check out repository code
+#        uses: ClickHouse/checkout@v1
+#      - name: Post
+#        run: |
+#          cd "$GITHUB_WORKSPACE/tests/ci"
+#          python3 auto_release.py --post-auto-release-complete --wf-status ${{ job.status }}

.github/workflows/create_release.yml

@@ -47,6 +47,8 @@ concurrency:
         required: false
         default: false
         type: boolean
+    secrets:
+      ROBOT_CLICKHOUSE_COMMIT_TOKEN:
 
 jobs:
   CreateRelease:

tests/ci/artifactory.py

@@ -143,6 +143,8 @@ class DebianArtifactory:
             print(f"  {cmd}")
             Shell.check(cmd, strict=True)
             Shell.check("sync")
+        time.sleep(10)
+        Shell.check(f"lsof +D R2MountPoint.MOUNT_POINT", verbose=True)
 
     def test_packages(self):
         Shell.check("docker pull ubuntu:latest", strict=True)

tests/ci/auto_release.py

@@ -46,6 +46,7 @@ def parse_args():
 
 MAX_NUMBER_OF_COMMITS_TO_CONSIDER_FOR_RELEASE = 5
 AUTORELEASE_INFO_FILE = "/tmp/autorelease_info.json"
+AUTORELEASE_MATRIX_PARAMS = "/tmp/autorelease_params.json"
 
 
 @dataclasses.dataclass
@@ -74,6 +75,14 @@ class AutoReleaseInfo:
         with open(AUTORELEASE_INFO_FILE, "w", encoding="utf-8") as f:
             print(json.dumps(dataclasses.asdict(self), indent=2), file=f)
 
+        # dump file for GH action matrix that is similar to the file above but with dropped not ready release branches
+        params = dataclasses.asdict(self)
+        params["releases"] = [
+            release for release in params["releases"] if release["ready"]
+        ]
+        with open(AUTORELEASE_MATRIX_PARAMS, "w", encoding="utf-8") as f:
+            print(json.dumps(params, indent=2), file=f)
+
     @staticmethod
     def from_file() -> "AutoReleaseInfo":
         with open(AUTORELEASE_INFO_FILE, "r", encoding="utf-8") as json_file:
@@ -102,7 +111,6 @@ def _prepare(token):
         refs = list(repo.get_git_matching_refs(f"tags/v{pr.head.ref}"))
         assert refs
 
-        refs.sort(key=lambda ref: ref.ref)
         latest_release_tag_ref = refs[-1]
         latest_release_tag = repo.get_git_tag(latest_release_tag_ref.object.sha)
 
@@ -110,6 +118,10 @@ def _prepare(token):
             f"git rev-list --first-parent {latest_release_tag.tag}..origin/{pr.head.ref}",
         ).split("\n")
         commit_num = len(commits)
+        if latest_release_tag.tag.endswith("new"):
+            print("It's a new release branch - skip auto release for it")
+            continue
+
         print(
             f"Previous release [{latest_release_tag.tag}] was [{commit_num}] commits ago, date [{latest_release_tag.tagger.date}]"
         )
@@ -133,16 +145,33 @@ def _prepare(token):
                 commits_to_branch_head += 1
                 continue
 
-            commit_ci_status = CI.GH.get_commit_status_by_name(
-                token=token,
-                commit_sha=commit,
-                status_name=(CI.JobNames.BUILD_CHECK, "ClickHouse build check"),
-            )
+            # TODO: switch to check if CI is entirely green
+            statuses = [
+                CI.GH.get_commit_status_by_name(
+                    token=token,
+                    commit_sha=commit,
+                    # handle old name for old releases
+                    status_name=(CI.JobNames.BUILD_CHECK, "ClickHouse build check"),
+                ),
+                CI.GH.get_commit_status_by_name(
+                    token=token,
+                    commit_sha=commit,
+                    # handle old name for old releases
+                    status_name=CI.JobNames.STATELESS_TEST_RELEASE,
+                ),
+                CI.GH.get_commit_status_by_name(
+                    token=token,
+                    commit_sha=commit,
+                    # handle old name for old releases
+                    status_name=CI.JobNames.STATEFUL_TEST_RELEASE,
+                ),
+            ]
             commit_sha = commit
-            if commit_ci_status == SUCCESS:
+            if any(status == SUCCESS for status in statuses):
+                commit_ci_status = SUCCESS
                 break
 
-            print(f"CI status [{commit_ci_status}] - skip")
+            print(f"CI status [{statuses}] - skip")
             commits_to_branch_head += 1
 
         ready = False

tests/ci/ci_utils.py

@@ -102,21 +102,29 @@ class GH:
         assert len(commit_sha) == 40
         assert Utils.is_hex(commit_sha)
         assert not Utils.is_hex(token)
-        url = f"https://api.github.com/repos/{Envs.GITHUB_REPOSITORY}/commits/{commit_sha}/statuses?per_page={200}"
+
+        url = f"https://api.github.com/repos/{Envs.GITHUB_REPOSITORY}/commits/{commit_sha}/statuses"
         headers = {
             "Authorization": f"token {token}",
             "Accept": "application/vnd.github.v3+json",
         }
-        response = requests.get(url, headers=headers, timeout=5)
 
         if isinstance(status_name, str):
             status_name = (status_name,)
-        if response.status_code == 200:
-            assert "next" not in response.links, "Response truncated"
-            statuses = response.json()
-            for status in statuses:
-                if status["context"] in status_name:
-                    return status["state"]  # type: ignore
+
+        while url:
+            response = requests.get(url, headers=headers, timeout=5)
+            if response.status_code == 200:
+                statuses = response.json()
+                for status in statuses:
+                    if status["context"] in status_name:
+                        return status["state"]  # type: ignore
+
+                # Check if there is a next page
+                url = response.links.get("next", {}).get("url")
+            else:
+                break
+
         return ""
 
     @staticmethod