CI: Sync, Merge check, CI gh's statuses fixes

.github/workflows/master.yml

@@ -27,15 +27,16 @@ jobs:
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 sync_pr.py --merge || :
-      - name: Python unit tests
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          echo "Testing the main ci directory"
-          python3 -m unittest discover -s . -p 'test_*.py'
-          for dir in *_lambda/; do
-            echo "Testing $dir"
-            python3 -m unittest discover -s "$dir" -p 'test_*.py'
-          done
+# Runs in MQ:
+#      - name: Python unit tests
+#        run: |
+#          cd "$GITHUB_WORKSPACE/tests/ci"
+#          echo "Testing the main ci directory"
+#          python3 -m unittest discover -s . -p 'test_*.py'
+#          for dir in *_lambda/; do
+#            echo "Testing $dir"
+#            python3 -m unittest discover -s "$dir" -p 'test_*.py'
+#          done
       - name: PrepareRunConfig
         id: runconfig
         run: |
@@ -162,7 +163,7 @@ jobs:
           python3 mark_release_ready.py
 
   FinishCheck:
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !cancelled() }}
     needs: [RunConfig, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3]
     runs-on: [self-hosted, style-checker-aarch64]
     steps:

.github/workflows/pull_request.yml

@@ -33,9 +33,12 @@ jobs:
           clear-repository: true # to ensure correct digests
           fetch-depth: 0 # to get a version
           filter: tree:0
-      - name: Cancel Sync PR workflow
+      - name: Cancel previous Sync PR workflow
         run: |
           python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --cancel-previous-run
+      - name: Set pending Sync status
+        run: |
+          python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --set-pending-status
       - name: Labels check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
@@ -177,7 +180,7 @@ jobs:
   ################################# Stage Final #################################
   #
   FinishCheck:
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !cancelled() }}
     needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3]
     runs-on: [self-hosted, style-checker-aarch64]
     steps:

tests/ci/ci.py

@@ -17,7 +17,7 @@ from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union
 import docker_images_helper
 import upload_result_helper
 from build_check import get_release_or_pr
-from ci_config import CI_CONFIG, Build, CILabels, CIStages, JobNames
+from ci_config import CI_CONFIG, Build, CILabels, CIStages, JobNames, StatusNames
 from ci_utils import GHActions, is_hex, normalize_string
 from clickhouse_helper import (
     CiLogsCredentials,
@@ -52,7 +52,7 @@ from git_helper import GIT_PREFIX, Git
 from git_helper import Runner as GitRunner
 from github_helper import GitHub
 from pr_info import PRInfo
-from report import ERROR, SUCCESS, BuildResult, JobReport
+from report import ERROR, SUCCESS, BuildResult, JobReport, PENDING
 from s3_helper import S3Helper
 from ci_metadata import CiMetadata
 from version_helper import get_version_from_repo
@@ -996,6 +996,11 @@ def parse_args(parser: argparse.ArgumentParser) -> argparse.Namespace:
         action="store_true",
         help="Action that cancels previous running PR workflow if PR added into the Merge Queue",
     )
+    parser.add_argument(
+        "--set-pending-status",
+        action="store_true",
+        help="Action to set needed pending statuses in the beginning of CI workflow, e.g. for Sync wf",
+    )
     parser.add_argument(
         "--configure",
         action="store_true",
@@ -1930,6 +1935,19 @@ def _cancel_pr_wf(s3: S3Helper, pr_number: int, cancel_sync: bool = False) -> No
             )
 
 
+def _set_pending_statuses(pr_info: PRInfo) -> None:
+    commit = get_commit(GitHub(get_best_robot_token(), per_page=100), pr_info.sha)
+    try:
+        commit.create_status(
+            state=PENDING,
+            target_url="",
+            description="",
+            context=StatusNames.SYNC,
+        )
+    except Exception as ex:
+        print(f"ERROR: failed to set GH commit status, ex: {ex}")
+
+
 def main() -> int:
     logging.basicConfig(level=logging.INFO)
     exit_code = 0
@@ -2265,6 +2283,13 @@ def main() -> int:
         else:
             assert False, "BUG! Not supported scenario"
 
+    ### SET PENDING STATUS
+    elif args.cancel_previous_run:
+        if pr_info.is_pr:
+            _set_pending_statuses(pr_info)
+        else:
+            assert False, "BUG! Not supported scenario"
+
     ### print results
     _print_results(result, args.outfile, args.pretty)
 

tests/ci/commit_status_helper.py

@@ -433,11 +433,8 @@ def set_mergeable_check(
     commit: Commit,
     description: str = "",
     state: StatusType = SUCCESS,
-    hide_url: bool = False,
 ) -> CommitStatus:
-    report_url = GITHUB_RUN_URL
-    if hide_url:
-        report_url = ""
+    report_url = ""
     return post_commit_status(
         commit,
         state,
@@ -469,7 +466,6 @@ def update_mergeable_check(commit: Commit, pr_info: PRInfo, check_name: str) ->
 def trigger_mergeable_check(
     commit: Commit,
     statuses: CommitStatuses,
-    hide_url: bool = False,
     set_if_green: bool = False,
     workflow_failed: bool = False,
 ) -> StatusType:
@@ -484,9 +480,12 @@ def trigger_mergeable_check(
 
     success = []
     fail = []
+    pending = []
     for status in required_checks:
         if status.state == SUCCESS:
             success.append(status.context)
+        elif status.state == PENDING:
+            pending.append(status.context)
         else:
             fail.append(status.context)
 
@@ -503,6 +502,8 @@ def trigger_mergeable_check(
     elif workflow_failed:
         description = "check workflow failures"
         state = FAILURE
+    elif pending:
+        description = "pending: " + ", ".join(pending)
     description = format_description(description)
 
     if not set_if_green and state == SUCCESS:
@@ -510,7 +511,7 @@ def trigger_mergeable_check(
         pass
     else:
         if mergeable_status is None or mergeable_status.description != description:
-            set_mergeable_check(commit, description, state, hide_url)
+            set_mergeable_check(commit, description, state)
 
     return state
 

tests/ci/finish_check.py

@@ -67,7 +67,7 @@ def main():
         if status.state == PENDING:
             post_commit_status(
                 commit,
-                SUCCESS,
+                state, # map Mergeable Check status to CI Running
                 status.target_url,
                 "All checks finished",
                 StatusNames.CI,

tests/ci/merge_pr.py

@@ -250,7 +250,6 @@ def main():
         trigger_mergeable_check(
             commit,
             statuses,
-            hide_url=False,
             set_if_green=True,
             workflow_failed=(args.wf_status != "success"),
         )