Add failed builds to the build report

.github/workflows/backport_branches.yml

@@ -359,15 +359,11 @@ jobs:
     steps:
       - name: Set envs
         run: |
-          DEPENDENCIES=$(cat << 'EOF' | jq '. | length'
-          ${{ toJSON(needs) }}
-          EOF
-          )
-          echo "DEPENDENCIES=$DEPENDENCIES" >> "$GITHUB_ENV"
           cat >> "$GITHUB_ENV" << 'EOF'
           CHECK_NAME=ClickHouse build check (actions)
           REPORTS_PATH=${{runner.temp}}/reports_dir
           TEMP_PATH=${{runner.temp}}/report_check
+          NEEDS_DATA_PATH=${{runner.temp}}/needs.json
           EOF
       - name: Download json reports
         uses: actions/download-artifact@v2
@@ -382,8 +378,11 @@ jobs:
         run: |
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
+          cat > "$NEEDS_DATA_PATH" << 'EOF'
+          ${{ toJSON(needs) }}
+          EOF
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 build_report_check.py "$CHECK_NAME" "$DEPENDENCIES"
+          python3 build_report_check.py "$CHECK_NAME"
       - name: Cleanup
         if: always()
         run: |

.github/workflows/master.yml

@@ -970,16 +970,12 @@ jobs:
     steps:
       - name: Set envs
         run: |
-          DEPENDENCIES=$(cat << 'EOF' | jq '. | length'
-          ${{ toJSON(needs) }}
-          EOF
-          )
-          echo "DEPENDENCIES=$DEPENDENCIES" >> "$GITHUB_ENV"
           cat >> "$GITHUB_ENV" << 'EOF'
           CHECK_NAME=ClickHouse build check (actions)
           REPORTS_PATH=${{runner.temp}}/reports_dir
           REPORTS_PATH=${{runner.temp}}/reports_dir
           TEMP_PATH=${{runner.temp}}/report_check
+          NEEDS_DATA_PATH=${{runner.temp}}/needs.json
           EOF
       - name: Download json reports
         uses: actions/download-artifact@v2
@@ -994,8 +990,11 @@ jobs:
         run: |
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
+          cat > "$NEEDS_DATA_PATH" << 'EOF'
+          ${{ toJSON(needs) }}
+          EOF
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 build_report_check.py "$CHECK_NAME" "$DEPENDENCIES"
+          python3 build_report_check.py "$CHECK_NAME"
       - name: Cleanup
         if: always()
         run: |
@@ -1018,15 +1017,11 @@ jobs:
     steps:
       - name: Set envs
         run: |
-          DEPENDENCIES=$(cat << 'EOF' | jq '. | length'
-          ${{ toJSON(needs) }}
-          EOF
-          )
-          echo "DEPENDENCIES=$DEPENDENCIES" >> "$GITHUB_ENV"
           cat >> "$GITHUB_ENV" << 'EOF'
           TEMP_PATH=${{runner.temp}}/report_check
           REPORTS_PATH=${{runner.temp}}/reports_dir
           CHECK_NAME=ClickHouse special build check (actions)
+          NEEDS_DATA_PATH=${{runner.temp}}/needs.json
           EOF
       - name: Download json reports
         uses: actions/download-artifact@v2
@@ -1041,8 +1036,11 @@ jobs:
         run: |
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
+          cat > "$NEEDS_DATA_PATH" << 'EOF'
+          ${{ toJSON(needs) }}
+          EOF
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 build_report_check.py "$CHECK_NAME" "$DEPENDENCIES"
+          python3 build_report_check.py "$CHECK_NAME"
       - name: Cleanup
         if: always()
         run: |

.github/workflows/pull_request.yml

@@ -1025,15 +1025,11 @@ jobs:
     steps:
       - name: Set envs
         run: |
-          DEPENDENCIES=$(cat << 'EOF' | jq '. | length'
-          ${{ toJSON(needs) }}
-          EOF
-          )
-          echo "DEPENDENCIES=$DEPENDENCIES" >> "$GITHUB_ENV"
           cat >> "$GITHUB_ENV" << 'EOF'
           CHECK_NAME=ClickHouse build check (actions)
           REPORTS_PATH=${{runner.temp}}/reports_dir
           TEMP_PATH=${{runner.temp}}/report_check
+          NEEDS_DATA_PATH=${{runner.temp}}/needs.json
           EOF
       - name: Download json reports
         uses: actions/download-artifact@v2
@@ -1048,8 +1044,11 @@ jobs:
         run: |
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
+          cat > "$NEEDS_DATA_PATH" << 'EOF'
+          ${{ toJSON(needs) }}
+          EOF
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 build_report_check.py "$CHECK_NAME" "$DEPENDENCIES"
+          python3 build_report_check.py "$CHECK_NAME"
       - name: Cleanup
         if: always()
         run: |
@@ -1073,15 +1072,11 @@ jobs:
     steps:
       - name: Set envs
         run: |
-          DEPENDENCIES=$(cat << 'EOF' | jq '. | length'
-          ${{ toJSON(needs) }}
-          EOF
-          )
-          echo "DEPENDENCIES=$DEPENDENCIES" >> "$GITHUB_ENV"
           cat >> "$GITHUB_ENV" << 'EOF'
           TEMP_PATH=${{runner.temp}}/report_check
           REPORTS_PATH=${{runner.temp}}/reports_dir
           CHECK_NAME=ClickHouse special build check (actions)
+          NEEDS_DATA_PATH=${{runner.temp}}/needs.json
           EOF
       - name: Download json reports
         uses: actions/download-artifact@v2
@@ -1096,8 +1091,11 @@ jobs:
         run: |
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
+          cat > "$NEEDS_DATA_PATH" << 'EOF'
+          ${{ toJSON(needs) }}
+          EOF
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 build_report_check.py "$CHECK_NAME" "$DEPENDENCIES"
+          python3 build_report_check.py "$CHECK_NAME"
       - name: Cleanup
         if: always()
         run: |

.github/workflows/release_branches.yml

@@ -442,16 +442,12 @@ jobs:
     steps:
       - name: Set envs
         run: |
-          DEPENDENCIES=$(cat << 'EOF' | jq '. | length'
-          ${{ toJSON(needs) }}
-          EOF
-          )
-          echo "DEPENDENCIES=$DEPENDENCIES" >> "$GITHUB_ENV"
           cat >> "$GITHUB_ENV" << 'EOF'
           CHECK_NAME=ClickHouse build check (actions)
           REPORTS_PATH=${{runner.temp}}/reports_dir
           REPORTS_PATH=${{runner.temp}}/reports_dir
           TEMP_PATH=${{runner.temp}}/report_check
+          NEEDS_DATA_PATH=${{runner.temp}}/needs.json
           EOF
       - name: Download json reports
         uses: actions/download-artifact@v2
@@ -466,8 +462,11 @@ jobs:
         run: |
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
+          cat > "$NEEDS_DATA_PATH" << 'EOF'
+          ${{ toJSON(needs) }}
+          EOF
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 build_report_check.py "$CHECK_NAME" "$DEPENDENCIES"
+          python3 build_report_check.py "$CHECK_NAME"
       - name: Cleanup
         if: always()
         run: |

tests/ci/build_check.py

@@ -9,7 +9,7 @@ import time
 from shutil import rmtree
 from typing import List, Optional, Tuple
 
-from env_helper import REPO_COPY, TEMP_PATH, CACHES_PATH, IMAGES_PATH
+from env_helper import GITHUB_JOB, REPO_COPY, TEMP_PATH, CACHES_PATH, IMAGES_PATH
 from s3_helper import S3Helper
 from pr_info import PRInfo
 from version_helper import (
@@ -138,6 +138,7 @@ def create_json_artifact(
         "build_config": build_config,
         "elapsed_seconds": elapsed,
         "status": success,
+        "job_name": GITHUB_JOB,
     }
 
     json_name = "build_urls_" + build_name + ".json"

tests/ci/build_report_check.py

@@ -9,11 +9,11 @@ from typing import Dict, List, Tuple
 from github import Github
 
 from env_helper import (
+    GITHUB_REPOSITORY,
+    GITHUB_RUN_URL,
+    GITHUB_SERVER_URL,
     REPORTS_PATH,
     TEMP_PATH,
-    GITHUB_REPOSITORY,
-    GITHUB_SERVER_URL,
-    GITHUB_RUN_URL,
 )
 from report import create_build_html_report
 from s3_helper import S3Helper
@@ -24,6 +24,9 @@ from ci_config import CI_CONFIG
 from rerun_helper import RerunHelper
 
 
+NEEDS_DATA_PATH = os.getenv("NEEDS_DATA_PATH")
+
+
 class BuildResult:
     def __init__(
         self,
@@ -76,6 +79,23 @@ def group_by_artifacts(build_urls: List[str]) -> Dict[str, List[str]]:
     return groups
 
 
+def get_failed_report(
+    job_name: str,
+) -> Tuple[List[BuildResult], List[List[str]], List[str]]:
+    message = f"{job_name} failed"
+    build_result = BuildResult(
+        compiler="unknown",
+        build_type="unknown",
+        sanitizer="unknown",
+        bundled="unknown",
+        splitted="unknown",
+        status=message,
+        elapsed_seconds=0,
+        with_coverage=False,
+    )
+    return [build_result], [[""]], [""]
+
+
 def process_report(
     build_report,
 ) -> Tuple[List[BuildResult], List[List[str]], List[str]]:
@@ -117,15 +137,19 @@ def get_build_name_from_file_name(file_name):
 
 def main():
     logging.basicConfig(level=logging.INFO)
-    reports_path = REPORTS_PATH
     temp_path = TEMP_PATH
-    logging.info("Reports path %s", reports_path)
+    logging.info("Reports path %s", REPORTS_PATH)
 
     if not os.path.exists(temp_path):
         os.makedirs(temp_path)
 
     build_check_name = sys.argv[1]
-    required_builds = int(sys.argv[2]) if len(sys.argv) > 2 else 0
+    needs_data = None
+    required_builds = 0
+    if os.path.exists(NEEDS_DATA_PATH):
+        with open(NEEDS_DATA_PATH, "rb") as file_handler:
+            needs_data = json.load(file_handler)
+            required_builds = len(needs_data)
 
     gh = Github(get_best_robot_token())
     pr_info = PRInfo()
@@ -140,7 +164,7 @@ def main():
 
     # Collect reports from json artifacts
     builds_report_map = {}
-    for root, _, files in os.walk(reports_path):
+    for root, _, files in os.walk(REPORTS_PATH):
         for f in files:
             if f.startswith("build_urls_") and f.endswith(".json"):
                 logging.info("Found build report json %s", f)
@@ -163,12 +187,18 @@ def main():
     ]
 
     some_builds_are_missing = len(build_reports) < required_builds
+    missing_build_names = []
     if some_builds_are_missing:
         logging.warning(
             "Expected to get %s build results, got only %s",
             required_builds,
             len(build_reports),
         )
+        missing_build_names = [
+            name
+            for name in needs_data
+            if not any(rep for rep in build_reports if rep["job_name"] == name)
+        ]
     else:
         logging.info("Got exactly %s builds", len(builds_report_map))
 
@@ -186,6 +216,14 @@ def main():
         build_artifacts.extend(build_artifacts_url)
         build_logs.extend(build_logs_url)
 
+    for failed_job in missing_build_names:
+        build_result, build_artifacts_url, build_logs_url = get_failed_report(
+            failed_job
+        )
+        build_results.extend(build_result)
+        build_artifacts.extend(build_artifacts_url)
+        build_logs.extend(build_logs_url)
+
     total_groups = len(build_results)
     logging.info("Totally got %s artifact groups", total_groups)
     if total_groups == 0:

tests/ci/env_helper.py

@@ -5,11 +5,12 @@ module_dir = p.abspath(p.dirname(__file__))
 git_root = p.abspath(p.join(module_dir, "..", ".."))
 
 CI = bool(os.getenv("CI"))
-TEMP_PATH = os.getenv("TEMP_PATH", module_dir)
+TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp")))
 
 CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH)
 CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN")
 GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "")
+GITHUB_JOB = os.getenv("GITHUB_JOB", "local")
 GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse")
 GITHUB_RUN_ID = os.getenv("GITHUB_RUN_ID", "0")
 GITHUB_SERVER_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com")