Merge pull request #46736 from ClickHouse/pending-on-rerun

Mark failed build reports as pending on reruns

tests/ci/build_check.py

@@ -9,6 +9,10 @@ import time
 from shutil import rmtree
 from typing import List, Tuple
 
+from ccache_utils import get_ccache_if_not_exists, upload_ccache
+from ci_config import CI_CONFIG, BuildConfig
+from commit_status_helper import get_commit_filtered_statuses, get_commit
+from docker_pull_helper import get_image_with_version
 from env_helper import (
     CACHES_PATH,
     GITHUB_JOB,
@@ -18,18 +22,17 @@ from env_helper import (
     S3_DOWNLOAD,
     TEMP_PATH,
 )
-from s3_helper import S3Helper
+from get_robot_token import get_best_robot_token
+from github_helper import GitHub
 from pr_info import PRInfo
+from s3_helper import S3Helper
+from tee_popen import TeePopen
 from version_helper import (
     ClickHouseVersion,
     Git,
     get_version_from_repo,
     update_version_local,
 )
-from ccache_utils import get_ccache_if_not_exists, upload_ccache
-from ci_config import CI_CONFIG, BuildConfig
-from docker_pull_helper import get_image_with_version
-from tee_popen import TeePopen
 
 IMAGE_NAME = "clickhouse/binary-builder"
 BUILD_LOG_NAME = "build_log.log"
@@ -122,8 +125,7 @@ def check_for_success_run(
     logged_prefix = os.path.join(S3_BUILDS_BUCKET, s3_prefix, "")
     logging.info("Checking for artifacts in %s", logged_prefix)
     try:
-        # TODO: theoretically, it would miss performance artifact for pr==0,
-        # but luckily we rerun only really failed tasks now, so we're safe
+        # Performance artifacts are now part of regular build, so we're safe
         build_results = s3_helper.list_prefix(s3_prefix)
     except Exception as ex:
         logging.info("Got exception while listing %s: %s\nRerun", logged_prefix, ex)
@@ -231,6 +233,29 @@ def upload_master_static_binaries(
     print(f"::notice ::Binary static URL: {url}")
 
 
+def mark_failed_reports_pending(build_name: str, sha: str) -> None:
+    try:
+        gh = GitHub(get_best_robot_token())
+        commit = get_commit(gh, sha)
+        statuses = get_commit_filtered_statuses(commit)
+        report_status = [
+            name
+            for name, builds in CI_CONFIG["builds_report_config"].items()
+            if build_name in builds
+        ][0]
+        for status in statuses:
+            if status.context == report_status and status.state in ["failure", "error"]:
+                logging.info(
+                    "Commit already have failed status for '%s', setting it to 'pending'",
+                    report_status,
+                )
+                commit.create_status(
+                    "pending", status.url, "Set to pending on rerun", report_status
+                )
+    except:  # we do not care about any exception here
+        logging.info("Failed to get or mark the reports status as pending, continue")
+
+
 def main():
     logging.basicConfig(level=logging.INFO)
 
@@ -260,6 +285,9 @@ def main():
     # put them as github actions artifact (result)
     check_for_success_run(s3_helper, s3_path_prefix, build_name, build_config)
 
+    # If it's a latter running, we need to mark possible failed status
+    mark_failed_reports_pending(build_name, pr_info.sha)
+
     docker_image = get_image_with_version(IMAGES_PATH, IMAGE_NAME)
     image_version = docker_image.version
 

tests/ci/cherry_pick.py

@@ -508,7 +508,7 @@ def main():
         logging.getLogger("git_helper").setLevel(logging.DEBUG)
     token = args.token or get_best_robot_token()
 
-    gh = GitHub(token, create_cache_dir=False, per_page=100)
+    gh = GitHub(token, create_cache_dir=False)
     bp = Backport(gh, args.repo, args.dry_run)
     # https://github.com/python/mypy/issues/3004
     bp.gh.cache_path = f"{TEMP_PATH}/gh_cache"  # type: ignore

tests/ci/commit_status_helper.py

@@ -6,11 +6,12 @@ import time
 from typing import List
 import logging
 
-from ci_config import CI_CONFIG, REQUIRED_CHECKS
-from env_helper import GITHUB_REPOSITORY, GITHUB_RUN_URL
 from github import Github
 from github.Commit import Commit
 from github.CommitStatus import CommitStatus
+
+from ci_config import CI_CONFIG, REQUIRED_CHECKS
+from env_helper import GITHUB_REPOSITORY, GITHUB_RUN_URL
 from pr_info import PRInfo, SKIP_MERGEABLE_CHECK_LABEL
 
 RETRY = 5

tests/ci/github_helper.py

@@ -35,6 +35,8 @@ class GitHub(github.Github):
         self._cache_path = Path(CACHE_PATH)
         if create_cache_dir:
             self.cache_path = self.cache_path
+        if not kwargs.get("per_page"):
+            kwargs["per_page"] = 100
         # And set Path
         super().__init__(*args, **kwargs)
         self._retries = 0

tests/ci/mark_release_ready.py

@@ -43,7 +43,7 @@ def main():
         description = "the release can be created from the commit"
         args.token = args.token or get_best_robot_token()
 
-    gh = GitHub(args.token, create_cache_dir=False, per_page=100)
+    gh = GitHub(args.token, create_cache_dir=False)
     # Get the rate limits for a quick fail
     gh.get_rate_limit()
     commit = get_commit(gh, args.commit)

tests/ci/merge_pr.py

@@ -217,7 +217,7 @@ def main():
     args = parse_args()
     logging.info("Going to process PR #%s in repo %s", args.pr, args.repo)
     token = args.token or get_best_robot_token()
-    gh = GitHub(token, per_page=100)
+    gh = GitHub(token)
     repo = gh.get_repo(args.repo)
     # An ugly and not nice fix to patch the wrong organization URL,
     # see https://github.com/PyGithub/PyGithub/issues/2395#issuecomment-1378629710

tests/ci/style_check.py

@@ -148,7 +148,7 @@ def main():
     if args.push:
         checkout_head(pr_info)
 
-    gh = GitHub(get_best_robot_token(), per_page=100, create_cache_dir=False)
+    gh = GitHub(get_best_robot_token(), create_cache_dir=False)
 
     atexit.register(update_mergeable_check, gh, pr_info, NAME)