Work around issues with GH callable actions

tests/ci/build_check.py

@@ -11,7 +11,7 @@ from ci_config import CI_CONFIG, BuildConfig
 from ccache_utils import CargoCache
 from docker_pull_helper import get_image_with_version
 from env_helper import (
-    GITHUB_JOB,
+    GITHUB_JOB_API_URL,
     IMAGES_PATH,
     REPO_COPY,
     S3_BUILDS_BUCKET,
@@ -163,7 +163,7 @@ def check_for_success_run(
         version.describe,
         SUCCESS if success else FAILURE,
         0,
-        GITHUB_JOB,
+        GITHUB_JOB_API_URL(),
     )
     build_result.write_json(Path(TEMP_PATH))
     # Fail build job if not successeded
@@ -348,7 +348,7 @@ def main():
         version.describe,
         build_status,
         elapsed,
-        GITHUB_JOB,
+        GITHUB_JOB_API_URL(),
     )
     result_json_path = build_result.write_json(temp_path)
     logging.info(

tests/ci/build_report_check.py

@@ -94,7 +94,7 @@ def main():
     missing_job_names = [
         name
         for name in needs_data
-        if not any(1 for build_result in build_results if build_result.job_name == name)
+        if not any(1 for br in build_results if br.job_name.startswith(name))
     ]
     missing_builds = len(missing_job_names)
     for job_name in reversed(missing_job_names):

tests/ci/env_helper.py

@@ -16,7 +16,7 @@ TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp")))
 CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH)
 CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN")
 GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "")
-GITHUB_JOB = os.getenv("GITHUB_JOB", "local")
+GITHUB_JOB = os.getenv("GITHUB_JOB_OVERRIDDEN", "") or os.getenv("GITHUB_JOB", "local")
 GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse")
 GITHUB_RUN_ID = os.getenv("GITHUB_RUN_ID", "0")
 GITHUB_SERVER_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com")
@@ -38,14 +38,16 @@ S3_ARTIFACT_DOWNLOAD_TEMPLATE = (
 # These parameters are set only on demand, and only once
 _GITHUB_JOB_ID = ""
 _GITHUB_JOB_URL = ""
+_GITHUB_JOB_API_URL = ""
 
 
 def GITHUB_JOB_ID() -> str:
     global _GITHUB_JOB_ID
     global _GITHUB_JOB_URL
+    global _GITHUB_JOB_API_URL
     if _GITHUB_JOB_ID:
         return _GITHUB_JOB_ID
-    _GITHUB_JOB_ID, _GITHUB_JOB_URL = get_job_id_url(GITHUB_JOB)
+    _GITHUB_JOB_ID, _GITHUB_JOB_URL, _GITHUB_JOB_API_URL = get_job_id_url(GITHUB_JOB)
     return _GITHUB_JOB_ID
 
 
@@ -54,13 +56,19 @@ def GITHUB_JOB_URL() -> str:
     return _GITHUB_JOB_URL
 
 
-def get_job_id_url(job_name: str) -> Tuple[str, str]:
+def GITHUB_JOB_API_URL() -> str:
+    GITHUB_JOB_ID()
+    return _GITHUB_JOB_API_URL
+
+
+def get_job_id_url(job_name: str) -> Tuple[str, str, str]:
     job_id = ""
     job_url = ""
+    job_api_url = ""
     if GITHUB_RUN_ID == "0":
         job_id = "0"
     if job_id:
-        return job_id, job_url
+        return job_id, job_url, job_api_url
     jobs = []
     page = 1
     while not job_id:
@@ -76,7 +84,8 @@ def get_job_id_url(job_name: str) -> Tuple[str, str]:
                 continue
             job_id = job["id"]
             job_url = job["html_url"]
-            return job_id, job_url
+            job_api_url = job["url"]
+            return job_id, job_url, job_api_url
         if (
             len(jobs) >= data["total_count"]  # just in case of inconsistency
             or len(data["jobs"]) == 0  # if we excided pages
@@ -100,7 +109,8 @@ def get_job_id_url(job_name: str) -> Tuple[str, str]:
             # The best case scenario
             job_id = matched_jobs[0]["id"]
             job_url = matched_jobs[0]["html_url"]
-            return job_id, job_url
+            job_api_url = matched_jobs[0]["url"]
+            return job_id, job_url, job_api_url
         if matched_jobs:
             logging.error(
                 "We could not get the ID and URL for the current job name %s, there "
@@ -109,4 +119,4 @@ def get_job_id_url(job_name: str) -> Tuple[str, str]:
                 job_name,
             )
 
-    return job_id, job_url
+    return job_id, job_url, job_api_url

tests/ci/report.py

@@ -10,8 +10,8 @@ import json
 import logging
 import os
 
+from build_download_helper import get_gh_api
 from ci_config import BuildConfig, CI_CONFIG
-from env_helper import get_job_id_url
 
 
 logger = logging.getLogger(__name__)
@@ -298,8 +298,10 @@ class BuildResult:
     version: str
     status: StatusType
     elapsed_seconds: int
-    job_name: str
-    _job_link: Optional[str] = None
+    job_api_url: str
+    _job_name: Optional[str] = None
+    _job_html_url: Optional[str] = None
+    _job_html_link: Optional[str] = None
     _grouped_urls: Optional[List[List[str]]] = None
 
     @property
@@ -387,11 +389,39 @@ class BuildResult:
 
     @property
     def job_link(self) -> str:
-        if self._job_link is not None:
-            return self._job_link
-        _, job_url = get_job_id_url(self.job_name)
-        self._job_link = f'<a href="{job_url}">{self.job_name}</a>'
-        return self._job_link
+        if self._job_html_link is not None:
+            return self._job_html_link
+        self._job_html_link = f'<a href="{self.job_html_url}">{self.job_name}</a>'
+        return self._job_html_link
+
+    @property
+    def job_html_url(self) -> str:
+        if self._job_html_url is not None:
+            return self._job_html_url
+        self._set_properties()
+        return self._job_html_url or ""
+
+    @property
+    def job_name(self) -> str:
+        if self._job_name is not None:
+            return self._job_name
+        self._set_properties()
+        return self._job_name or ""
+
+    @job_name.setter
+    def job_name(self, job_name: str) -> None:
+        self._job_name = job_name
+
+    def _set_properties(self) -> None:
+        if all(p is not None for p in (self._job_name, self._job_html_url)):
+            return
+        try:
+            job_data = get_gh_api(self.job_api_url).json()
+        except Exception:
+            job_data = {}
+        # job_name can be set manually
+        self._job_name = self._job_name or job_data.get("name", "unknown")
+        self._job_html_url = job_data.get("html_url", "")
 
     @staticmethod
     def get_report_name(name: str) -> Path:
@@ -416,7 +446,7 @@ class BuildResult:
             data.get("version", ""),
             data.get("status", ERROR),
             data.get("elapsed_seconds", 0),
-            data.get("job_name", ""),
+            data.get("job_api_url", ""),
         )
 
     @staticmethod
@@ -434,7 +464,7 @@ class BuildResult:
                     "version": self.version,
                     "status": self.status,
                     "elapsed_seconds": self.elapsed_seconds,
-                    "job_name": self.job_name,
+                    "job_api_url": self.job_api_url,
                 }
             ),
             encoding="utf-8",