From 4d336d0228da8523ddccf037a04e5b3acaf30911 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 31 May 2022 17:09:36 +0000 Subject: [PATCH] Get rid of duplicate download_previous_release --- docker/test/stress/Dockerfile | 1 - docker/test/stress/download_previous_release | 110 ------------------- docker/test/stress/run.sh | 2 +- tests/ci/bugfix_validate_check.py | 2 - tests/ci/download_previous_release.py | 29 +++-- tests/ci/stress_check.py | 4 + 6 files changed, 19 insertions(+), 129 deletions(-) delete mode 100755 docker/test/stress/download_previous_release mode change 100644 => 100755 tests/ci/download_previous_release.py diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index ba6daffc014..393508fd551 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -25,7 +25,6 @@ RUN apt-get update -y \ brotli COPY ./stress /stress -COPY ./download_previous_release /download_previous_release COPY run.sh / ENV DATASETS="hits visits" diff --git a/docker/test/stress/download_previous_release b/docker/test/stress/download_previous_release deleted file mode 100755 index ea3d376ad90..00000000000 --- a/docker/test/stress/download_previous_release +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python3 - -import requests -import re -import os - -from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry - -CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags" - -CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb" -CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static-dbg_{version}_amd64.deb" -CLICKHOUSE_SERVER_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-server_{version}_all.deb" -CLICKHOUSE_CLIENT_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-client_{version}_amd64.deb" - - -CLICKHOUSE_COMMON_STATIC_PACKET_NAME = "clickhouse-common-static_{version}_amd64.deb" -CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME = "clickhouse-common-static-dbg_{version}_amd64.deb" -CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_all.deb" -CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_all.deb" - -PACKETS_DIR = "previous_release_package_folder/" -VERSION_PATTERN = r"((?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)" - - -class Version: - def __init__(self, version): - self.version = version - - def __lt__(self, other): - return list(map(int, self.version.split('.'))) < list(map(int, other.version.split('.'))) - - def __str__(self): - return self.version - - -class ReleaseInfo: - def __init__(self, version, release_type): - self.version = version - self.type = release_type - - -def find_previous_release(server_version, releases): - releases.sort(key=lambda x: x.version, reverse=True) - for release in releases: - if release.version < server_version: - return True, release - - return False, None - - -def get_previous_release(server_version): - page = 1 - found = False - while not found: - response = requests.get(CLICKHOUSE_TAGS_URL, {'page': page, 'per_page': 100}) - if not response.ok: - raise Exception('Cannot load the list of tags from github: ' + response.reason) - - releases_str = set(re.findall(VERSION_PATTERN, response.text)) - if len(releases_str) == 0: - raise Exception('Cannot find previous release for ' + str(server_version) + ' server version') - - releases = list(map(lambda x: ReleaseInfo(Version(x.split('-')[0]), x.split('-')[1]), releases_str)) - found, previous_release = find_previous_release(server_version, releases) - page += 1 - - return previous_release - - -def download_packet(url, local_file_name, retries=10, backoff_factor=0.3): - session = requests.Session() - retry = Retry( - total=retries, - read=retries, - connect=retries, - backoff_factor=backoff_factor, - ) - adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) - response = session.get(url) - print(url) - if response.ok: - open(PACKETS_DIR + local_file_name, 'wb').write(response.content) - - -def download_packets(release): - if not os.path.exists(PACKETS_DIR): - os.makedirs(PACKETS_DIR) - - download_packet(CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format(version=release.version, type=release.type), - CLICKHOUSE_COMMON_STATIC_PACKET_NAME.format(version=release.version)) - - download_packet(CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL.format(version=release.version, type=release.type), - CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME.format(version=release.version)) - - download_packet(CLICKHOUSE_SERVER_DOWNLOAD_URL.format(version=release.version, type=release.type), - CLICKHOUSE_SERVER_PACKET_NAME.format(version=release.version)) - - download_packet(CLICKHOUSE_CLIENT_DOWNLOAD_URL.format(version=release.version, type=release.type), - CLICKHOUSE_CLIENT_PACKET_NAME.format(version=release.version)) - - -if __name__ == '__main__': - server_version = Version(input()) - previous_release = get_previous_release(server_version) - download_packets(previous_release) - diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 97b9225a2d2..af789a645ff 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -262,7 +262,7 @@ echo -e "Backward compatibility check\n" echo "Download previous release server" mkdir previous_release_package_folder -clickhouse-client --query="SELECT version()" | ./download_previous_release && echo -e 'Download script exit code\tOK' >> /test_output/test_results.tsv \ +clickhouse-client --query="SELECT version()" | download_previous_release && echo -e 'Download script exit code\tOK' >> /test_output/test_results.tsv \ || echo -e 'Download script failed\tFAIL' >> /test_output/test_results.tsv stop diff --git a/tests/ci/bugfix_validate_check.py b/tests/ci/bugfix_validate_check.py index 7c130a766a1..4e6001aaa74 100644 --- a/tests/ci/bugfix_validate_check.py +++ b/tests/ci/bugfix_validate_check.py @@ -6,8 +6,6 @@ import itertools import os import sys -NO_CHANGES_MSG = "Nothing to run" - def parse_args(): parser = argparse.ArgumentParser() diff --git a/tests/ci/download_previous_release.py b/tests/ci/download_previous_release.py old mode 100644 new mode 100755 index 16d0f9e4939..fa03d164f23 --- a/tests/ci/download_previous_release.py +++ b/tests/ci/download_previous_release.py @@ -1,18 +1,14 @@ #!/usr/bin/env python3 -########################################################################### -# # -# TODO (@vdimir, @Avogar) # -# Merge with one from https://github.com/ClickHouse/ClickHouse/pull/27928 # -# # -########################################################################### - import re import os import logging import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags" CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb" @@ -67,7 +63,7 @@ def find_previous_release(server_version, releases): return False, None -def get_previous_release(server_version=None): +def get_previous_release(server_version): page = 1 found = False while not found: @@ -97,13 +93,16 @@ def get_previous_release(server_version=None): return previous_release -def download_packet(url, out_path): - """ - TODO: use dowload_build_with_progress from build_download_helper.py - """ - - response = requests.get(url) - logging.info("Downloading %s", url) +def download_packet(url, out_path, retries=10, backoff_factor=0.3): + session = requests.Session() + retry = Retry( + total=retries, read=retries, connect=retries, backoff_factor=backoff_factor + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount("http://", adapter) + session.mount("https://", adapter) + response = session.get(url) + print(url) if response.ok: open(out_path, "wb").write(response.content) diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 2c5ec2cdc05..bcfbceb4496 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -29,6 +29,9 @@ from tee_popen import TeePopen def get_run_command( build_path, result_folder, repo_tests_path, server_log_folder, image ): + download_previous_release = os.path.join( + repo_tests_path, "ci/download_previous_release.py" + ) cmd = ( "docker run --cap-add=SYS_PTRACE " "-e S3_URL='https://clickhouse-datasets.s3.amazonaws.com' " @@ -36,6 +39,7 @@ def get_run_command( f"--volume={result_folder}:/test_output " f"--volume={repo_tests_path}:/usr/share/clickhouse-test " f"--volume={server_log_folder}:/var/log/clickhouse-server {image}" + f"--volume={download_previous_release}:/usr/local/bin/download_previous_release " ) return cmd