From 4947d6db13952b9d73d22a2236ddddec957f584d Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 9 Mar 2022 16:24:58 +0000 Subject: [PATCH] Use download_previous_release for bugfix validation Should be merged with https://github.com/ClickHouse/ClickHouse/pull/27928 --- .github/workflows/pull_request.yml | 2 +- tests/ci/download_previous_release.py | 126 ++++++++++++++++++++++++++ tests/ci/functional_test_check.py | 9 +- 3 files changed, 133 insertions(+), 4 deletions(-) create mode 100644 tests/ci/download_previous_release.py diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index f9883546c23..ae6d4c5aaef 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1734,7 +1734,7 @@ jobs: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestBugfixCheck: - needs: [BuilderDebAsan] + needs: [] runs-on: [self-hosted, func-tester] steps: - name: Set envs diff --git a/tests/ci/download_previous_release.py b/tests/ci/download_previous_release.py new file mode 100644 index 00000000000..b11a84a36f6 --- /dev/null +++ b/tests/ci/download_previous_release.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 + +########################################################################### +# # +# TODO (@vdimir, @Avogar) # +# Merge with one from https://github.com/ClickHouse/ClickHouse/pull/27928 # +# # +########################################################################### + +import requests +import re +import os + +CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags" + +CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb" +CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static-dbg_{version}_amd64.deb" +CLICKHOUSE_SERVER_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-server_{version}_all.deb" +CLICKHOUSE_CLIENT_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-client_{version}_amd64.deb" + + +CLICKHOUSE_COMMON_STATIC_PACKET_NAME = "clickhouse-common-static_{version}_amd64.deb" +CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME = "clickhouse-common-static-dbg_{version}_amd64.deb" +CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_all.deb" +CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_all.deb" + +PACKETS_DIR = "previous_release_package_folder/" +VERSION_PATTERN = r"((?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)" + + +class Version: + def __init__(self, version): + self.version = version + + def __lt__(self, other): + return list(map(int, self.version.split('.'))) < list(map(int, other.version.split('.'))) + + def __str__(self): + return self.version + + +class ReleaseInfo: + def __init__(self, version, release_type): + self.version = version + self.type = release_type + + def __repr__(self): + return f"ReleaseInfo: {self.version}-{self.type}" + +def find_previous_release(server_version, releases): + releases.sort(key=lambda x: x.version, reverse=True) + print(releases) + if server_version is None: + return True, releases[0] + + for release in releases: + if release.version < server_version: + return True, release + + return False, None + + +def get_previous_release(server_version=None): + page = 1 + found = False + while not found: + response = requests.get(CLICKHOUSE_TAGS_URL, {'page': page, 'per_page': 100}) + if not response.ok: + raise Exception('Cannot load the list of tags from github: ' + response.reason) + + releases_str = set(re.findall(VERSION_PATTERN, response.text)) + if len(releases_str) == 0: + raise Exception('Cannot find previous release for ' + str(server_version) + ' server version') + + releases = list(map(lambda x: ReleaseInfo(Version(x.split('-')[0]), x.split('-')[1]), releases_str)) + found, previous_release = find_previous_release(server_version, releases) + page += 1 + + return previous_release + + +def download_packet(url, out_path): + """ + TODO: use dowload_build_with_progress from build_download_helper.py + """ + + response = requests.get(url) + print(url) + if response.ok: + open(out_path, 'wb').write(response.content) + +def download_packets(release, dest_path=PACKETS_DIR): + if not os.path.exists(dest_path): + os.makedirs(dest_path) + + download_packet( + CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format(version=release.version, type=release.type), + out_path=os.path.join(dest_path, CLICKHOUSE_COMMON_STATIC_PACKET_NAME.format(version=release.version)), + ) + + download_packet( + CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL.format(version=release.version, type=release.type), + out_path=os.path.join(dest_path, CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME.format(version=release.version)), + ) + + download_packet( + CLICKHOUSE_SERVER_DOWNLOAD_URL.format(version=release.version, type=release.type), + out_path=os.path.join(dest_path, CLICKHOUSE_SERVER_PACKET_NAME.format(version=release.version)), + ) + + download_packet( + CLICKHOUSE_CLIENT_DOWNLOAD_URL.format(version=release.version, type=release.type), + out_path=os.path.join(dest_path, CLICKHOUSE_CLIENT_PACKET_NAME.format(version=release.version)), + ) + + +def download_previous_release(dest_path): + current_release = get_previous_release(None) + download_packets(current_release, dest_path=dest_path) + + +if __name__ == '__main__': + server_version = Version(input()) + previous_release = get_previous_release(server_version) + download_packets(previous_release) + diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index efec6c03278..7890673384a 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -13,6 +13,7 @@ from s3_helper import S3Helper from get_robot_token import get_best_robot_token from pr_info import PRInfo from build_download_helper import download_all_deb_packages +from download_previous_release import download_previous_release from upload_result_helper import upload_results from docker_pull_helper import get_image_with_version from commit_status_helper import post_commit_status, get_commit, override_status @@ -173,8 +174,7 @@ if __name__ == "__main__": sys.exit(0) image_name = get_image_name(check_name) - image_ver = 0 if validate_bugix_check else None - docker_image = get_image_with_version(reports_path, image_name, version=image_ver) + docker_image = get_image_with_version(reports_path, image_name) repo_tests_path = os.path.join(repo_path, "tests") @@ -182,7 +182,10 @@ if __name__ == "__main__": if not os.path.exists(packages_path): os.makedirs(packages_path) - download_all_deb_packages(check_name, reports_path, packages_path) + if not validate_bugix_check: + download_all_deb_packages(check_name, reports_path, packages_path) + else: + download_previous_release(packages_path) server_log_path = os.path.join(temp_path, "server_log") if not os.path.exists(server_log_path):