mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 00:30:49 +00:00
Add type checking to build_download_helper
This commit is contained in:
parent
6b43d2e197
commit
4efddb5ed4
@ -6,7 +6,7 @@ import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, List, Union
|
||||
from typing import Any, Callable, List, Optional, Union
|
||||
|
||||
# isort: off
|
||||
import requests
|
||||
@ -185,18 +185,21 @@ def download_build_with_progress(url: str, path: Path) -> None:
|
||||
|
||||
|
||||
def download_builds(
|
||||
result_path: str, build_urls: List[str], filter_fn: Callable[[str], bool]
|
||||
result_path: Path, build_urls: List[str], filter_fn: Callable[[str], bool]
|
||||
) -> None:
|
||||
for url in build_urls:
|
||||
if filter_fn(url):
|
||||
fname = os.path.basename(url.replace("%2B", "+").replace("%20", " "))
|
||||
logging.info("Will download %s to %s", fname, result_path)
|
||||
download_build_with_progress(url, Path(result_path) / fname)
|
||||
download_build_with_progress(url, result_path / fname)
|
||||
|
||||
|
||||
def download_builds_filter(
|
||||
check_name, reports_path, result_path, filter_fn=lambda _: True
|
||||
):
|
||||
check_name: str,
|
||||
reports_path: Union[Path, str],
|
||||
result_path: Path,
|
||||
filter_fn: Callable[[str], bool] = lambda _: True,
|
||||
) -> None:
|
||||
build_name = get_build_name_for_check(check_name)
|
||||
urls = read_build_urls(build_name, reports_path)
|
||||
logging.info("The build report for %s contains the next URLs: %s", build_name, urls)
|
||||
@ -207,25 +210,33 @@ def download_builds_filter(
|
||||
download_builds(result_path, urls, filter_fn)
|
||||
|
||||
|
||||
def download_all_deb_packages(check_name, reports_path, result_path):
|
||||
def download_all_deb_packages(
|
||||
check_name: str, reports_path: Union[Path, str], result_path: Path
|
||||
) -> None:
|
||||
download_builds_filter(
|
||||
check_name, reports_path, result_path, lambda x: x.endswith("deb")
|
||||
)
|
||||
|
||||
|
||||
def download_unit_tests(check_name, reports_path, result_path):
|
||||
def download_unit_tests(
|
||||
check_name: str, reports_path: Union[Path, str], result_path: Path
|
||||
) -> None:
|
||||
download_builds_filter(
|
||||
check_name, reports_path, result_path, lambda x: x.endswith("unit_tests_dbms")
|
||||
)
|
||||
|
||||
|
||||
def download_clickhouse_binary(check_name, reports_path, result_path):
|
||||
def download_clickhouse_binary(
|
||||
check_name: str, reports_path: Union[Path, str], result_path: Path
|
||||
) -> None:
|
||||
download_builds_filter(
|
||||
check_name, reports_path, result_path, lambda x: x.endswith("clickhouse")
|
||||
)
|
||||
|
||||
|
||||
def get_clickhouse_binary_url(check_name, reports_path):
|
||||
def get_clickhouse_binary_url(
|
||||
check_name: str, reports_path: Union[Path, str]
|
||||
) -> Optional[str]:
|
||||
build_name = get_build_name_for_check(check_name)
|
||||
urls = read_build_urls(build_name, reports_path)
|
||||
logging.info("The build report for %s contains the next URLs: %s", build_name, urls)
|
||||
@ -240,7 +251,9 @@ def get_clickhouse_binary_url(check_name, reports_path):
|
||||
return None
|
||||
|
||||
|
||||
def download_performance_build(check_name, reports_path, result_path):
|
||||
def download_performance_build(
|
||||
check_name: str, reports_path: Union[Path, str], result_path: Path
|
||||
) -> None:
|
||||
download_builds_filter(
|
||||
check_name,
|
||||
reports_path,
|
||||
@ -249,7 +262,9 @@ def download_performance_build(check_name, reports_path, result_path):
|
||||
)
|
||||
|
||||
|
||||
def download_fuzzers(check_name, reports_path, result_path):
|
||||
def download_fuzzers(
|
||||
check_name: str, reports_path: Union[Path, str], result_path: Path
|
||||
) -> None:
|
||||
download_builds_filter(
|
||||
check_name,
|
||||
reports_path,
|
||||
|
@ -155,7 +155,7 @@ def main():
|
||||
}
|
||||
|
||||
download_builds_filter(
|
||||
check_name, REPORT_PATH, TEMP_PATH, lambda url: "performance.tar.zst" in url
|
||||
check_name, REPORT_PATH, temp_path, lambda url: "performance.tar.zst" in url
|
||||
)
|
||||
assert os.path.exists(f"{TEMP_PATH}/performance.tar.zst"), "Perf artifact not found"
|
||||
|
||||
|
@ -166,7 +166,7 @@ def main():
|
||||
|
||||
docker_image = pull_image(get_docker_image(IMAGE_NAME))
|
||||
|
||||
download_unit_tests(check_name, REPORT_PATH, TEMP_PATH)
|
||||
download_unit_tests(check_name, REPORT_PATH, temp_path)
|
||||
|
||||
tests_binary = temp_path / "unit_tests_dbms"
|
||||
os.chmod(tests_binary, 0o777)
|
||||
|
Loading…
Reference in New Issue
Block a user