thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 17:12:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use a named logger in build_download_helper

Browse Source
This commit is contained in:
Mikhail f. Shiryaev 2024-06-07 17:07:06 +02:00
parent 0dff60821f
commit 8c4f5c65aa
No known key found for this signature in database
GPG Key ID: 4B02ED204C7D93F4
1 changed files with 18 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
tests/ci/build_download_helper.py
View File

@ -15,6 +15,8 @@ from ci_config import CI_CONFIG
DOWNLOAD_RETRIES_COUNT = 5 DOWNLOAD_RETRIES_COUNT = 5
logger = logging.getLogger(__name__)
class DownloadException(Exception): class DownloadException(Exception):
pass pass
@ -30,7 +32,7 @@ def get_with_retries(
sleep: int = 3, sleep: int = 3,
**kwargs: Any, **kwargs: Any,
) -> requests.Response: ) -> requests.Response:
logging.info( logger.info(
"Getting URL with %i tries and sleep %i in between: %s", retries, sleep, url "Getting URL with %i tries and sleep %i in between: %s", retries, sleep, url
) )
exc = Exception("A placeholder to satisfy typing and avoid nesting") exc = Exception("A placeholder to satisfy typing and avoid nesting")
@ -42,7 +44,7 @@ def get_with_retries(
return response return response
except Exception as e: except Exception as e:
if i + 1 < retries: if i + 1 < retries:
logging.info("Exception '%s' while getting, retry %i", e, i + 1) logger.info("Exception '%s' while getting, retry %i", e, i + 1)
time.sleep(sleep) time.sleep(sleep)
exc = e exc = e
@ -96,7 +98,7 @@ def get_gh_api(
) )
try_auth = e.response.status_code == 404 try_auth = e.response.status_code == 404
if (ratelimit_exceeded or try_auth) and not token_is_set: if (ratelimit_exceeded or try_auth) and not token_is_set:
logging.warning( logger.warning(
"Received rate limit exception, setting the auth header and retry" "Received rate limit exception, setting the auth header and retry"
) )
set_auth_header() set_auth_header()
@ -107,7 +109,7 @@ def get_gh_api(
exc = e exc = e
if try_cnt < retries: if try_cnt < retries:
logging.info("Exception '%s' while getting, retry %i", exc, try_cnt) logger.info("Exception '%s' while getting, retry %i", exc, try_cnt)
time.sleep(sleep) time.sleep(sleep)
raise APIException(f"Unable to request data from GH API: {url}") from exc raise APIException(f"Unable to request data from GH API: {url}") from exc
@ -121,25 +123,25 @@ def read_build_urls(build_name: str, reports_path: Union[Path, str]) -> List[str
for root, _, files in os.walk(reports_path): for root, _, files in os.walk(reports_path):
for file in files: for file in files:
if file.endswith(f"_{build_name}.json"): if file.endswith(f"_{build_name}.json"):
logging.info("Found build report json %s for %s", file, build_name) logger.info("Found build report json %s for %s", file, build_name)
with open( with open(
os.path.join(root, file), "r", encoding="utf-8" os.path.join(root, file), "r", encoding="utf-8"
) as file_handler: ) as file_handler:
build_report = json.load(file_handler) build_report = json.load(file_handler)
return build_report["build_urls"] # type: ignore return build_report["build_urls"] # type: ignore
logging.info("A build report is not found for %s", build_name) logger.info("A build report is not found for %s", build_name)
return [] return []
def download_build_with_progress(url: str, path: Path) -> None: def download_build_with_progress(url: str, path: Path) -> None:
logging.info("Downloading from %s to temp path %s", url, path) logger.info("Downloading from %s to temp path %s", url, path)
for i in range(DOWNLOAD_RETRIES_COUNT): for i in range(DOWNLOAD_RETRIES_COUNT):
try: try:
response = get_with_retries(url, retries=1, stream=True) response = get_with_retries(url, retries=1, stream=True)
total_length = int(response.headers.get("content-length", 0)) total_length = int(response.headers.get("content-length", 0))
if path.is_file() and total_length and path.stat().st_size == total_length: if path.is_file() and total_length and path.stat().st_size == total_length:
logging.info( logger.info(
"The file %s already exists and have a proper size %s", "The file %s already exists and have a proper size %s",
path, path,
total_length, total_length,
@ -148,14 +150,14 @@ def download_build_with_progress(url: str, path: Path) -> None:
with open(path, "wb") as f: with open(path, "wb") as f:
if total_length == 0: if total_length == 0:
logging.info( logger.info(
"No content-length, will download file without progress" "No content-length, will download file without progress"
) )
f.write(response.content) f.write(response.content)
else: else:
dl = 0 dl = 0
logging.info("Content length is %ld bytes", total_length) logger.info("Content length is %ld bytes", total_length)
for data in response.iter_content(chunk_size=4096): for data in response.iter_content(chunk_size=4096):
dl += len(data) dl += len(data)
f.write(data) f.write(data)
@ -170,8 +172,8 @@ def download_build_with_progress(url: str, path: Path) -> None:
except Exception as e: except Exception as e:
if sys.stdout.isatty(): if sys.stdout.isatty():
sys.stdout.write("\n") sys.stdout.write("\n")
if os.path.exists(path): if path.exists():
os.remove(path) path.unlink()
if i + 1 < DOWNLOAD_RETRIES_COUNT: if i + 1 < DOWNLOAD_RETRIES_COUNT:
time.sleep(3) time.sleep(3)
@ -182,7 +184,7 @@ def download_build_with_progress(url: str, path: Path) -> None:
if sys.stdout.isatty(): if sys.stdout.isatty():
sys.stdout.write("\n") sys.stdout.write("\n")
logging.info("Downloading finished") logger.info("Downloading finished")
def download_builds( def download_builds(
@ -191,7 +193,7 @@ def download_builds(
for url in build_urls: for url in build_urls:
if filter_fn(url): if filter_fn(url):
fname = os.path.basename(url.replace("%2B", "+").replace("%20", " ")) fname = os.path.basename(url.replace("%2B", "+").replace("%20", " "))
logging.info("Will download %s to %s", fname, result_path) logger.info("Will download %s to %s", fname, result_path)
download_build_with_progress(url, result_path / fname) download_build_with_progress(url, result_path / fname)
@ -203,7 +205,7 @@ def download_builds_filter(
) -> None: ) -> None:
build_name = get_build_name_for_check(check_name) build_name = get_build_name_for_check(check_name)
urls = read_build_urls(build_name, reports_path) urls = read_build_urls(build_name, reports_path)
logging.info("The build report for %s contains the next URLs: %s", build_name, urls) logger.info("The build report for %s contains the next URLs: %s", build_name, urls)
if not urls: if not urls:
raise DownloadException("No build URLs found") raise DownloadException("No build URLs found")
@ -240,7 +242,7 @@ def get_clickhouse_binary_url(
) -> Optional[str]: ) -> Optional[str]:
build_name = get_build_name_for_check(check_name) build_name = get_build_name_for_check(check_name)
urls = read_build_urls(build_name, reports_path) urls = read_build_urls(build_name, reports_path)
logging.info("The build report for %s contains the next URLs: %s", build_name, urls) logger.info("The build report for %s contains the next URLs: %s", build_name, urls)
for url in urls: for url in urls:
check_url = url check_url = url
if "?" in check_url: if "?" in check_url: