Merge pull request #61592 from ClickHouse/improve-builds-downloader

Improve build_download_helper

tests/ci/build_download_helper.py

@@ -6,7 +6,7 @@ import os
 import sys
 import time
 from pathlib import Path
-from typing import Any, Callable, List, Union
+from typing import Any, Callable, List, Optional, Union
 
 # isort: off
 import requests
@@ -120,12 +120,14 @@ def read_build_urls(build_name: str, reports_path: Union[Path, str]) -> List[str
     for root, _, files in os.walk(reports_path):
         for file in files:
             if file.endswith(f"_{build_name}.json"):
-                logging.info("Found build report json %s", file)
+                logging.info("Found build report json %s for %s", file, build_name)
                 with open(
                     os.path.join(root, file), "r", encoding="utf-8"
                 ) as file_handler:
                     build_report = json.load(file_handler)
                     return build_report["build_urls"]  # type: ignore
+
+    logging.info("A build report is not found for %s", build_name)
     return []
 
 
@@ -183,21 +185,24 @@ def download_build_with_progress(url: str, path: Path) -> None:
 
 
 def download_builds(
-    result_path: str, build_urls: List[str], filter_fn: Callable[[str], bool]
+    result_path: Path, build_urls: List[str], filter_fn: Callable[[str], bool]
 ) -> None:
     for url in build_urls:
         if filter_fn(url):
             fname = os.path.basename(url.replace("%2B", "+").replace("%20", " "))
             logging.info("Will download %s to %s", fname, result_path)
-            download_build_with_progress(url, Path(result_path) / fname)
+            download_build_with_progress(url, result_path / fname)
 
 
 def download_builds_filter(
-    check_name, reports_path, result_path, filter_fn=lambda _: True
-):
+    check_name: str,
+    reports_path: Union[Path, str],
+    result_path: Path,
+    filter_fn: Callable[[str], bool] = lambda _: True,
+) -> None:
     build_name = get_build_name_for_check(check_name)
     urls = read_build_urls(build_name, reports_path)
-    print(urls)
+    logging.info("The build report for %s contains the next URLs: %s", build_name, urls)
 
     if not urls:
         raise DownloadException("No build URLs found")
@@ -205,25 +210,50 @@ def download_builds_filter(
     download_builds(result_path, urls, filter_fn)
 
 
-def download_all_deb_packages(check_name, reports_path, result_path):
+def download_all_deb_packages(
+    check_name: str, reports_path: Union[Path, str], result_path: Path
+) -> None:
     download_builds_filter(
         check_name, reports_path, result_path, lambda x: x.endswith("deb")
     )
 
 
-def download_unit_tests(check_name, reports_path, result_path):
+def download_unit_tests(
+    check_name: str, reports_path: Union[Path, str], result_path: Path
+) -> None:
     download_builds_filter(
         check_name, reports_path, result_path, lambda x: x.endswith("unit_tests_dbms")
     )
 
 
-def download_clickhouse_binary(check_name, reports_path, result_path):
+def download_clickhouse_binary(
+    check_name: str, reports_path: Union[Path, str], result_path: Path
+) -> None:
     download_builds_filter(
         check_name, reports_path, result_path, lambda x: x.endswith("clickhouse")
     )
 
 
-def download_performance_build(check_name, reports_path, result_path):
+def get_clickhouse_binary_url(
+    check_name: str, reports_path: Union[Path, str]
+) -> Optional[str]:
+    build_name = get_build_name_for_check(check_name)
+    urls = read_build_urls(build_name, reports_path)
+    logging.info("The build report for %s contains the next URLs: %s", build_name, urls)
+    for url in urls:
+        check_url = url
+        if "?" in check_url:
+            check_url = check_url.split("?")[0]
+
+        if check_url.endswith("clickhouse"):
+            return url
+
+    return None
+
+
+def download_performance_build(
+    check_name: str, reports_path: Union[Path, str], result_path: Path
+) -> None:
     download_builds_filter(
         check_name,
         reports_path,
@@ -232,7 +262,9 @@ def download_performance_build(check_name, reports_path, result_path):
     )
 
 
-def download_fuzzers(check_name, reports_path, result_path):
+def download_fuzzers(
+    check_name: str, reports_path: Union[Path, str], result_path: Path
+) -> None:
     download_builds_filter(
         check_name,
         reports_path,

tests/ci/performance_comparison_check.py

@@ -155,7 +155,7 @@ def main():
     }
 
     download_builds_filter(
-        check_name, REPORT_PATH, TEMP_PATH, lambda url: "performance.tar.zst" in url
+        check_name, REPORT_PATH, temp_path, lambda url: "performance.tar.zst" in url
     )
     assert os.path.exists(f"{TEMP_PATH}/performance.tar.zst"), "Perf artifact not found"
 

tests/ci/unit_tests_check.py

@@ -166,7 +166,7 @@ def main():
 
     docker_image = pull_image(get_docker_image(IMAGE_NAME))
 
-    download_unit_tests(check_name, REPORT_PATH, TEMP_PATH)
+    download_unit_tests(check_name, REPORT_PATH, temp_path)
 
     tests_binary = temp_path / "unit_tests_dbms"
     os.chmod(tests_binary, 0o777)