Backport #71782 to 24.3: Fix checks DB and reports for edge cases

tests/ci/ci.py

@@ -2077,6 +2077,7 @@ def main() -> int:
                         s3,
                         pr_info.number,
                         pr_info.sha,
+                        pr_info.head_ref,
                         job_report.test_results,
                         job_report.additional_files,
                         job_report.check_name or _get_ext_check_name(args.job_name),
@@ -2114,6 +2115,69 @@ def main() -> int:
                 db="default", table="checks", events=prepared_events
             )
         else:
+<<<<<<< HEAD
+            print("ERROR: Job was killed - generate evidence")
+            job_report.update_duration()
+            ret_code = os.getenv("JOB_EXIT_CODE", "")
+            if ret_code:
+                try:
+                    job_report.exit_code = int(ret_code)
+                except ValueError:
+                    pass
+            if Utils.is_killed_with_oom():
+                print("WARNING: OOM while job execution")
+                print(subprocess.run("sudo dmesg -T", check=False))
+                error_description = f"Out Of Memory, exit_code {job_report.exit_code}"
+            else:
+                error_description = f"Unknown, exit_code {job_report.exit_code}"
+            CIBuddy().post_job_error(
+                error_description + f" after {int(job_report.duration)}s",
+                job_name=_get_ext_check_name(args.job_name),
+            )
+            if CI.is_test_job(args.job_name):
+                gh = GitHub(get_best_robot_token(), per_page=100)
+                commit = get_commit(gh, pr_info.sha)
+                check_url = ""
+                if job_report.test_results or job_report.additional_files:
+                    check_url = upload_result_helper.upload_results(
+                        s3,
+                        pr_info.number,
+                        pr_info.sha,
+                        pr_info.head_ref,
+                        job_report.test_results,
+                        job_report.additional_files,
+                        job_report.check_name or _get_ext_check_name(args.job_name),
+                    )
+                post_commit_status(
+                    commit,
+                    ERROR,
+                    check_url,
+                    "Error: " + error_description,
+                    _get_ext_check_name(args.job_name),
+                    pr_info,
+                    dump_to_file=True,
+                )
+
+        if not job_report.job_skipped:
+            print("push finish record to ci db")
+            prepared_events = prepare_tests_results_for_clickhouse(
+                pr_info,
+                [
+                    TestResult(
+                        JOB_FINISHED_TEST_NAME,
+                        FAIL if error_description else OK,
+                        raw_logs=error_description or None,
+                    )
+                ],
+                SUCCESS if not error_description else ERROR,
+                0.0,
+                JobReport.get_start_time_from_current(),
+                "",
+                _get_ext_check_name(args.job_name),
+            )
+            ClickHouseHelper().insert_events_into(
+                db="default", table="checks", events=prepared_events
+            )
             # no job report
             print(f"No job report for {[args.job_name]} - do nothing")
     ### POST action: end

tests/ci/clickhouse_helper.py

@@ -7,6 +7,8 @@ from pathlib import Path
 from typing import Dict, List, Optional
 
 import requests
+
+from env_helper import GITHUB_REPOSITORY
 from get_robot_token import get_parameter_from_ssm
 from pr_info import PRInfo
 from report import TestResults
@@ -204,17 +206,13 @@ def prepare_tests_results_for_clickhouse(
     report_url: str,
     check_name: str,
 ) -> List[dict]:
-    pull_request_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
+    base_ref = pr_info.base_ref
-    base_ref = "master"
+    base_repo = pr_info.base_name
-    head_ref = "master"
+    head_ref = pr_info.head_ref
-    base_repo = pr_info.repo_full_name
+    head_repo = pr_info.head_name
-    head_repo = pr_info.repo_full_name
+    pull_request_url = f"https://github.com/{GITHUB_REPOSITORY}/commits/{head_ref}"
     if pr_info.number != 0:
         pull_request_url = pr_info.pr_html_url
-        base_ref = pr_info.base_ref
-        base_repo = pr_info.base_name
-        head_ref = pr_info.head_ref
-        head_repo = pr_info.head_name
 
     common_properties = {
         "pull_request_number": pr_info.number,

tests/ci/commit_status_helper.py

@@ -294,7 +294,13 @@ def create_ci_report(pr_info: PRInfo, statuses: CommitStatuses) -> str:
             )
         )
     return upload_results(
-        S3Helper(), pr_info.number, pr_info.sha, test_results, [], CI_STATUS_NAME
+        S3Helper(),
+        pr_info.number,
+        pr_info.sha,
+        pr_info.head_ref,
+        test_results,
+        [],
+        CI_STATUS_NAME,
     )
 
 

tests/ci/docker_images_check.py

@@ -250,7 +250,9 @@ def main():
     s3_helper = S3Helper()
 
     pr_info = PRInfo()
-    url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [], NAME)
+    url = upload_results(
+        s3_helper, pr_info.number, pr_info.sha, pr_info.head_ref, test_results, [], NAME
+    )
 
     print(f"::notice ::Report url: {url}")
 

tests/ci/docker_manifests_merge.py

@@ -183,7 +183,9 @@ def main():
     pr_info = PRInfo()
     s3_helper = S3Helper()
 
-    url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [], NAME)
+    url = upload_results(
+        s3_helper, pr_info.number, pr_info.sha, pr_info.head_ref, test_results, [], NAME
+    )
 
     print(f"::notice ::Report url: {url}")
 

tests/ci/pr_info.py

@@ -121,6 +121,12 @@ class PRInfo:
         ref = github_event.get("ref", "refs/heads/master")
         if ref and ref.startswith("refs/heads/"):
             ref = ref[11:]
+        # Default values
+        self.base_ref = ""  # type: str
+        self.base_name = ""  # type: str
+        self.head_ref = ""  # type: str
+        self.head_name = ""  # type: str
+        self.number = 0  # type: int
 
         # workflow completed event, used for PRs only
         if "action" in github_event and github_event["action"] == "completed":
@@ -135,7 +141,7 @@ class PRInfo:
 
         if "pull_request" in github_event:  # pull request and other similar events
             self.event_type = EventType.PULL_REQUEST
-            self.number = github_event["pull_request"]["number"]  # type: int
+            self.number = github_event["pull_request"]["number"]
             if pr_event_from_api:
                 try:
                     response = get_gh_api(
@@ -161,17 +167,13 @@ class PRInfo:
             self.pr_html_url = f"{repo_prefix}/pull/{self.number}"
 
             # master or backport/xx.x/xxxxx - where the PR will be merged
-            self.base_ref = github_event["pull_request"]["base"]["ref"]  # type: str
+            self.base_ref = github_event["pull_request"]["base"]["ref"]
             # ClickHouse/ClickHouse
-            self.base_name = github_event["pull_request"]["base"]["repo"][
+            self.base_name = github_event["pull_request"]["base"]["repo"]["full_name"]
-                "full_name"
-            ]  # type: str
             # any_branch-name - the name of working branch name
-            self.head_ref = github_event["pull_request"]["head"]["ref"]  # type: str
+            self.head_ref = github_event["pull_request"]["head"]["ref"]
             # UserName/ClickHouse or ClickHouse/ClickHouse
-            self.head_name = github_event["pull_request"]["head"]["repo"][
+            self.head_name = github_event["pull_request"]["head"]["repo"]["full_name"]
-                "full_name"
-            ]  # type: str
             self.body = github_event["pull_request"]["body"]
             self.labels = {
                 label["name"] for label in github_event["pull_request"]["labels"]

tests/ci/upload_result_helper.py

@@ -57,6 +57,7 @@ def upload_results(
     s3_client: S3Helper,
     pr_number: int,
     commit_sha: str,
+    branch_name: str,
     test_results: TestResults,
     additional_files: Union[Sequence[Path], Sequence[str]],
     check_name: str,
@@ -73,8 +74,7 @@ def upload_results(
         process_logs(s3_client, additional_files, s3_path_prefix, test_results)
     )
 
-    branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commits/master"
+    branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commits/{branch_name}"
-    branch_name = "master"
     if pr_number != 0:
         branch_name = f"PR #{pr_number}"
         branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/pull/{pr_number}"