CI: POC for Auto Releases

.github/workflows/auto_release.yml

@@ -1,44 +1,58 @@
 name: AutoRelease
 
 env:
-  # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
 
 concurrency:
-  group: auto-release
+  group: release
 on: # yamllint disable-line rule:truthy
   # schedule:
   #   - cron: '0 10-16 * * 1-5'
   workflow_dispatch:
 
 jobs:
-  CherryPick:
-    runs-on: [self-hosted, style-checker-aarch64]
+  AutoRelease:
+    runs-on: [self-hosted, release-maker]
     steps:
+      - name: DebugInfo
+        uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
       - name: Set envs
-        # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings
         run: |
           cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/cherry_pick
           ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
           ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
           RCSK
-          REPO_OWNER=ClickHouse
-          REPO_NAME=ClickHouse
-          REPO_TEAM=core
           EOF
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
         with:
-          clear-repository: true
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
           fetch-depth: 0
-      - name: Auto-release
+      - name: Auto Release Prepare
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 auto_release.py --release-after-days=3
-      - name: Cleanup
-        if: always()
+          python3 auto_release.py
+          echo "::group::Auto Release Info"
+          python3 -m json.tool /tmp/autorelease_info.json
+          echo "::endgroup::"
+          {
+              echo 'AUTO_RELEASE_PARAMS<<EOF'
+              cat  /tmp/autorelease_info.json
+              echo 'EOF'
+          } >> "$GITHUB_ENV"
+      - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0].release_branch }}
+        if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0] }}
+        uses: ./.github/workflows/create_release.yml
+        with:
+          type: patch
+          ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0].commit_sha }}
+          dry-run: true
+          autorelease: true
+      - name: Post Slack Message
+        if: ${{ !cancelled() }}
+        run: |
+          echo Slack Message
+      - name: Clean up
         run: |
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:

.github/workflows/create_release.yml

@@ -22,6 +22,10 @@ concurrency:
         required: false
         default: true
         type: boolean
+      autorelease:
+        required: false
+        default: false
+        type: boolean
 
 jobs:
   CreateRelease:
@@ -30,8 +34,10 @@ jobs:
     runs-on: [self-hosted, release-maker]
     steps:
       - name: DebugInfo
+        if: ${{ ! inputs.autorelease }}
         uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
       - name: Set envs
+        if: ${{ ! inputs.autorelease }}
         # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings
         run: |
           cat >> "$GITHUB_ENV" << 'EOF'
@@ -41,6 +47,7 @@ jobs:
           RELEASE_INFO_FILE=${{ runner.temp }}/release_info.json
           EOF
       - name: Check out repository code
+        if: ${{ ! inputs.autorelease }}
         uses: ClickHouse/checkout@v1
         with:
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
@@ -161,6 +168,6 @@ jobs:
           export CHECK_NAME="Docker keeper image"
           python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
       - name: Post Slack Message
-        if: always()
+        if: ${{ !cancelled() }}
         run: |
           echo Slack Message

tests/ci/auto_release.py

@@ -1,13 +1,16 @@
 import argparse
-from datetime import timedelta, datetime
+import dataclasses
+import json
 import logging
 import os
-from commit_status_helper import get_commit_filtered_statuses
+from typing import List
+
 from get_robot_token import get_best_robot_token
 from github_helper import GitHub
-from release import Release, Repo as ReleaseRepo, RELEASE_READY_STATUS
-from report import SUCCESS
 from ssh import SSHKey
+from ci_utils import Shell
+from env_helper import GITHUB_REPOSITORY
+from report import SUCCESS
 
 LOGGER_NAME = __name__
 HELPER_LOGGERS = ["github_helper", LOGGER_NAME]
@@ -20,116 +23,104 @@ def parse_args():
         "branches and do a release in case for green builds."
     )
     parser.add_argument("--token", help="GitHub token, if not set, used from smm")
-    parser.add_argument(
-        "--repo", default="ClickHouse/ClickHouse", help="Repo owner/name"
-    )
-    parser.add_argument("--dry-run", action="store_true", help="Do not create anything")
-    parser.add_argument(
-        "--release-after-days",
-        type=int,
-        default=3,
-        help="Do automatic release on the latest green commit after the latest "
-        "release if the newest release is older than the specified days",
-    )
-    parser.add_argument(
-        "--debug-helpers",
-        action="store_true",
-        help="Add debug logging for this script and github_helper",
-    )
-    parser.add_argument(
-        "--remote-protocol",
-        "-p",
-        default="ssh",
-        choices=ReleaseRepo.VALID,
-        help="repo protocol for git commands remote, 'origin' is a special case and "
-        "uses 'origin' as a remote",
-    )
 
     return parser.parse_args()
 
 
+MAX_NUMBER_OF_COMMITS_TO_CONSIDER_FOR_RELEASE = 5
+AUTORELEASE_INFO_FILE = "/tmp/autorelease_info.json"
+
+
+@dataclasses.dataclass
+class ReleaseParams:
+    release_branch: str
+    commit_sha: str
+
+
+@dataclasses.dataclass
+class AutoReleaseInfo:
+    releases: List[ReleaseParams]
+
+    def add_release(self, release_params: ReleaseParams):
+        self.releases.append(release_params)
+
+    def dump(self):
+        print(f"Dump release info into [{AUTORELEASE_INFO_FILE}]")
+        with open(AUTORELEASE_INFO_FILE, "w", encoding="utf-8") as f:
+            print(json.dumps(dataclasses.asdict(self), indent=2), file=f)
+
+
 def main():
     args = parse_args()
-    logging.basicConfig(level=logging.INFO)
-    if args.debug_helpers:
-        for logger_name in HELPER_LOGGERS:
-            logging.getLogger(logger_name).setLevel(logging.DEBUG)
 
     token = args.token or get_best_robot_token()
-    days_as_timedelta = timedelta(days=args.release_after_days)
-    now = datetime.now()
-
+    assert len(token) > 10
+    os.environ["GH_TOKEN"] = token
+    (Shell.run("gh auth status", check=True))
     gh = GitHub(token)
-    prs = gh.get_release_pulls(args.repo)
+    prs = gh.get_release_pulls(GITHUB_REPOSITORY)
     branch_names = [pr.head.ref for pr in prs]
 
-    logger.info("Found release branches: %s\n ", " \n".join(branch_names))
-    repo = gh.get_repo(args.repo)
+    print(f"Found release branches [{branch_names}]")
+    repo = gh.get_repo(GITHUB_REPOSITORY)
 
-    # In general there is no guarantee on which order the refs/commits are
-    # returned from the API, so we have to order them.
+    autoRelease_info = AutoReleaseInfo(releases=[])
     for pr in prs:
-        logger.info("Checking PR %s", pr.head.ref)
+        print(f"Checking PR [{pr.head.ref}]")
 
         refs = list(repo.get_git_matching_refs(f"tags/v{pr.head.ref}"))
         refs.sort(key=lambda ref: ref.ref)
 
         latest_release_tag_ref = refs[-1]
         latest_release_tag = repo.get_git_tag(latest_release_tag_ref.object.sha)
-        logger.info("That last release was done at %s", latest_release_tag.tagger.date)
-
-        if latest_release_tag.tagger.date + days_as_timedelta > now:
-            logger.info(
-                "Not enough days since the last release %s,"
-                " no automatic release can be done",
-                latest_release_tag.tag,
+        commit_num = int(
+            Shell.run(
+                f"git rev-list --count {latest_release_tag.tag}..origin/{pr.head.ref}",
+                check=True,
             )
-            continue
-
-        unreleased_commits = list(
-            repo.get_commits(sha=pr.head.ref, since=latest_release_tag.tagger.date)
         )
-        unreleased_commits.sort(
-            key=lambda commit: commit.commit.committer.date, reverse=True
+        print(
+            f"Previous release is [{latest_release_tag}] was [{commit_num}] commits before, date [{latest_release_tag.tagger.date}]"
         )
-
-        for commit in unreleased_commits:
-            logger.info("Checking statuses of commit %s", commit.sha)
-            statuses = get_commit_filtered_statuses(commit)
-            all_success = all(st.state == SUCCESS for st in statuses)
-            passed_ready_for_release_check = any(
-                st.context == RELEASE_READY_STATUS and st.state == SUCCESS
-                for st in statuses
+        commit_reverse_index = 0
+        commit_found = False
+        commit_checked = False
+        commit_sha = ""
+        while (
+            commit_reverse_index < commit_num - 1
+            and commit_reverse_index < MAX_NUMBER_OF_COMMITS_TO_CONSIDER_FOR_RELEASE
+        ):
+            commit_checked = True
+            commit_sha = Shell.run(
+                f"git rev-list --max-count=1 --skip={commit_reverse_index} origin/{pr.head.ref}",
+                check=True,
             )
-            if not (all_success and passed_ready_for_release_check):
-                logger.info("Commit is not green, thus not suitable for release")
-                continue
-
-            logger.info("Commit is ready for release, let's release!")
-
-            release = Release(
-                ReleaseRepo(args.repo, args.remote_protocol),
-                commit.sha,
-                "patch",
-                args.dry_run,
-                True,
+            print(
+                f"Check if commit [{commit_sha}] [{pr.head.ref}~{commit_reverse_index}] is ready for release"
             )
-            try:
-                release.do(True, True, True)
-            except:
-                if release.has_rollback:
-                    logging.error(
-                        "!!The release process finished with error, read the output carefully!!"
-                    )
-                    logging.error(
-                        "Probably, rollback finished with error. "
-                        "If you don't see any of the following commands in the output, "
-                        "execute them manually:"
-                    )
-                    release.log_rollback()
-                raise
-            logging.info("New release is done!")
+            commit_reverse_index += 1
+
+            cmd = f"gh api -H 'Accept: application/vnd.github.v3+json' /repos/{GITHUB_REPOSITORY}/commits/{commit_sha}/status"
+            ci_status_json = Shell.run(cmd, check=True)
+            ci_status = json.loads(ci_status_json)["state"]
+            if ci_status == SUCCESS:
+                commit_found = True
             break
+        if commit_found:
+            print(
+                f"Add release ready info for commit [{commit_sha}] and release branch [{pr.head.ref}]"
+            )
+            autoRelease_info.add_release(
+                ReleaseParams(release_branch=pr.head.ref, commit_sha=commit_sha)
+            )
+        else:
+            print(f"WARNING: No good commits found for release branch [{pr.head.ref}]")
+            if commit_checked:
+                print(
+                    f"ERROR: CI is failed. check CI status for branch [{pr.head.ref}]"
+                )
+
+    autoRelease_info.dump()
 
 
 if __name__ == "__main__":

tests/ci/ci_utils.py

@@ -95,7 +95,8 @@ class Shell:
         return res.stdout.strip()
 
     @classmethod
-    def run(cls, command):
+    def run(cls, command, check=False):
+        print(f"Run command [{command}]")
         res = ""
         result = subprocess.run(
             command,
@@ -107,6 +108,9 @@ class Shell:
         )
         if result.returncode == 0:
             res = result.stdout
+        elif check:
+            print(f"ERROR: stdout {result.stdout}, stderr {result.stderr}")
+            assert result.returncode == 0
         return res.strip()
 
     @classmethod