Add type checking to build_download_helper

This commit is contained in:
Mikhail f. Shiryaev 2024-03-19 14:01:31 +01:00
parent 6b43d2e197
commit 4efddb5ed4
No known key found for this signature in database
GPG Key ID: 4B02ED204C7D93F4
3 changed files with 28 additions and 13 deletions

View File

@ -6,7 +6,7 @@ import os
import sys
import time
from pathlib import Path
from typing import Any, Callable, List, Union
from typing import Any, Callable, List, Optional, Union
# isort: off
import requests
@ -185,18 +185,21 @@ def download_build_with_progress(url: str, path: Path) -> None:
def download_builds(
result_path: str, build_urls: List[str], filter_fn: Callable[[str], bool]
result_path: Path, build_urls: List[str], filter_fn: Callable[[str], bool]
) -> None:
for url in build_urls:
if filter_fn(url):
fname = os.path.basename(url.replace("%2B", "+").replace("%20", " "))
logging.info("Will download %s to %s", fname, result_path)
download_build_with_progress(url, Path(result_path) / fname)
download_build_with_progress(url, result_path / fname)
def download_builds_filter(
check_name, reports_path, result_path, filter_fn=lambda _: True
):
check_name: str,
reports_path: Union[Path, str],
result_path: Path,
filter_fn: Callable[[str], bool] = lambda _: True,
) -> None:
build_name = get_build_name_for_check(check_name)
urls = read_build_urls(build_name, reports_path)
logging.info("The build report for %s contains the next URLs: %s", build_name, urls)
@ -207,25 +210,33 @@ def download_builds_filter(
download_builds(result_path, urls, filter_fn)
def download_all_deb_packages(check_name, reports_path, result_path):
def download_all_deb_packages(
check_name: str, reports_path: Union[Path, str], result_path: Path
) -> None:
download_builds_filter(
check_name, reports_path, result_path, lambda x: x.endswith("deb")
)
def download_unit_tests(check_name, reports_path, result_path):
def download_unit_tests(
check_name: str, reports_path: Union[Path, str], result_path: Path
) -> None:
download_builds_filter(
check_name, reports_path, result_path, lambda x: x.endswith("unit_tests_dbms")
)
def download_clickhouse_binary(check_name, reports_path, result_path):
def download_clickhouse_binary(
check_name: str, reports_path: Union[Path, str], result_path: Path
) -> None:
download_builds_filter(
check_name, reports_path, result_path, lambda x: x.endswith("clickhouse")
)
def get_clickhouse_binary_url(check_name, reports_path):
def get_clickhouse_binary_url(
check_name: str, reports_path: Union[Path, str]
) -> Optional[str]:
build_name = get_build_name_for_check(check_name)
urls = read_build_urls(build_name, reports_path)
logging.info("The build report for %s contains the next URLs: %s", build_name, urls)
@ -240,7 +251,9 @@ def get_clickhouse_binary_url(check_name, reports_path):
return None
def download_performance_build(check_name, reports_path, result_path):
def download_performance_build(
check_name: str, reports_path: Union[Path, str], result_path: Path
) -> None:
download_builds_filter(
check_name,
reports_path,
@ -249,7 +262,9 @@ def download_performance_build(check_name, reports_path, result_path):
)
def download_fuzzers(check_name, reports_path, result_path):
def download_fuzzers(
check_name: str, reports_path: Union[Path, str], result_path: Path
) -> None:
download_builds_filter(
check_name,
reports_path,

View File

@ -155,7 +155,7 @@ def main():
}
download_builds_filter(
check_name, REPORT_PATH, TEMP_PATH, lambda url: "performance.tar.zst" in url
check_name, REPORT_PATH, temp_path, lambda url: "performance.tar.zst" in url
)
assert os.path.exists(f"{TEMP_PATH}/performance.tar.zst"), "Perf artifact not found"

View File

@ -166,7 +166,7 @@ def main():
docker_image = pull_image(get_docker_image(IMAGE_NAME))
download_unit_tests(check_name, REPORT_PATH, TEMP_PATH)
download_unit_tests(check_name, REPORT_PATH, temp_path)
tests_binary = temp_path / "unit_tests_dbms"
os.chmod(tests_binary, 0o777)