mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Get rid of duplicate download_previous_release
This commit is contained in:
vdimir
parent
2a38fdb796
commit
4d336d0228
@ -25,7 +25,6 @@ RUN apt-get update -y \
|
||||
brotli
|
||||
|
||||
COPY ./stress /stress
|
||||
COPY ./download_previous_release /download_previous_release
|
||||
COPY run.sh /
|
||||
|
||||
ENV DATASETS="hits visits"
|
||||
|
||||
@ -1,110 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import requests
|
||||
import re
|
||||
import os
|
||||
|
||||
from requests.adapters import HTTPAdapter
|
||||
from requests.packages.urllib3.util.retry import Retry
|
||||
|
||||
CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags"
|
||||
|
||||
CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb"
|
||||
CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static-dbg_{version}_amd64.deb"
|
||||
CLICKHOUSE_SERVER_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-server_{version}_all.deb"
|
||||
CLICKHOUSE_CLIENT_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-client_{version}_amd64.deb"
|
||||
|
||||
|
||||
CLICKHOUSE_COMMON_STATIC_PACKET_NAME = "clickhouse-common-static_{version}_amd64.deb"
|
||||
CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME = "clickhouse-common-static-dbg_{version}_amd64.deb"
|
||||
CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_all.deb"
|
||||
CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_all.deb"
|
||||
|
||||
PACKETS_DIR = "previous_release_package_folder/"
|
||||
VERSION_PATTERN = r"((?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)"
|
||||
|
||||
|
||||
class Version:
|
||||
def __init__(self, version):
|
||||
self.version = version
|
||||
|
||||
def __lt__(self, other):
|
||||
return list(map(int, self.version.split('.'))) < list(map(int, other.version.split('.')))
|
||||
|
||||
def __str__(self):
|
||||
return self.version
|
||||
|
||||
|
||||
class ReleaseInfo:
|
||||
def __init__(self, version, release_type):
|
||||
self.version = version
|
||||
self.type = release_type
|
||||
|
||||
|
||||
def find_previous_release(server_version, releases):
|
||||
releases.sort(key=lambda x: x.version, reverse=True)
|
||||
for release in releases:
|
||||
if release.version < server_version:
|
||||
return True, release
|
||||
|
||||
return False, None
|
||||
|
||||
|
||||
def get_previous_release(server_version):
|
||||
page = 1
|
||||
found = False
|
||||
while not found:
|
||||
response = requests.get(CLICKHOUSE_TAGS_URL, {'page': page, 'per_page': 100})
|
||||
if not response.ok:
|
||||
raise Exception('Cannot load the list of tags from github: ' + response.reason)
|
||||
|
||||
releases_str = set(re.findall(VERSION_PATTERN, response.text))
|
||||
if len(releases_str) == 0:
|
||||
raise Exception('Cannot find previous release for ' + str(server_version) + ' server version')
|
||||
|
||||
releases = list(map(lambda x: ReleaseInfo(Version(x.split('-')[0]), x.split('-')[1]), releases_str))
|
||||
found, previous_release = find_previous_release(server_version, releases)
|
||||
page += 1
|
||||
|
||||
return previous_release
|
||||
|
||||
|
||||
def download_packet(url, local_file_name, retries=10, backoff_factor=0.3):
|
||||
session = requests.Session()
|
||||
retry = Retry(
|
||||
total=retries,
|
||||
read=retries,
|
||||
connect=retries,
|
||||
backoff_factor=backoff_factor,
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry)
|
||||
session.mount('http://', adapter)
|
||||
session.mount('https://', adapter)
|
||||
response = session.get(url)
|
||||
print(url)
|
||||
if response.ok:
|
||||
open(PACKETS_DIR + local_file_name, 'wb').write(response.content)
|
||||
|
||||
|
||||
def download_packets(release):
|
||||
if not os.path.exists(PACKETS_DIR):
|
||||
os.makedirs(PACKETS_DIR)
|
||||
|
||||
download_packet(CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format(version=release.version, type=release.type),
|
||||
CLICKHOUSE_COMMON_STATIC_PACKET_NAME.format(version=release.version))
|
||||
|
||||
download_packet(CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL.format(version=release.version, type=release.type),
|
||||
CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME.format(version=release.version))
|
||||
|
||||
download_packet(CLICKHOUSE_SERVER_DOWNLOAD_URL.format(version=release.version, type=release.type),
|
||||
CLICKHOUSE_SERVER_PACKET_NAME.format(version=release.version))
|
||||
|
||||
download_packet(CLICKHOUSE_CLIENT_DOWNLOAD_URL.format(version=release.version, type=release.type),
|
||||
CLICKHOUSE_CLIENT_PACKET_NAME.format(version=release.version))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
server_version = Version(input())
|
||||
previous_release = get_previous_release(server_version)
|
||||
download_packets(previous_release)
|
||||
|
||||
@ -262,7 +262,7 @@ echo -e "Backward compatibility check\n"
|
||||
|
||||
echo "Download previous release server"
|
||||
mkdir previous_release_package_folder
|
||||
clickhouse-client --query="SELECT version()" | ./download_previous_release && echo -e 'Download script exit code\tOK' >> /test_output/test_results.tsv \
|
||||
clickhouse-client --query="SELECT version()" | download_previous_release && echo -e 'Download script exit code\tOK' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Download script failed\tFAIL' >> /test_output/test_results.tsv
|
||||
|
||||
stop
|
||||
|
||||
@ -6,8 +6,6 @@ import itertools
|
||||
import os
|
||||
import sys
|
||||
|
||||
NO_CHANGES_MSG = "Nothing to run"
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
29
tests/ci/download_previous_release.py
Normal file → Executable file
29
tests/ci/download_previous_release.py
Normal file → Executable file
@ -1,18 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
###########################################################################
|
||||
# #
|
||||
# TODO (@vdimir, @Avogar) #
|
||||
# Merge with one from https://github.com/ClickHouse/ClickHouse/pull/27928 #
|
||||
# #
|
||||
###########################################################################
|
||||
|
||||
import re
|
||||
import os
|
||||
import logging
|
||||
|
||||
import requests
|
||||
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags"
|
||||
|
||||
CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb"
|
||||
@ -67,7 +63,7 @@ def find_previous_release(server_version, releases):
|
||||
return False, None
|
||||
|
||||
|
||||
def get_previous_release(server_version=None):
|
||||
def get_previous_release(server_version):
|
||||
page = 1
|
||||
found = False
|
||||
while not found:
|
||||
@ -97,13 +93,16 @@ def get_previous_release(server_version=None):
|
||||
return previous_release
|
||||
|
||||
|
||||
def download_packet(url, out_path):
|
||||
"""
|
||||
TODO: use dowload_build_with_progress from build_download_helper.py
|
||||
"""
|
||||
|
||||
response = requests.get(url)
|
||||
logging.info("Downloading %s", url)
|
||||
def download_packet(url, out_path, retries=10, backoff_factor=0.3):
|
||||
session = requests.Session()
|
||||
retry = Retry(
|
||||
total=retries, read=retries, connect=retries, backoff_factor=backoff_factor
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry)
|
||||
session.mount("http://", adapter)
|
||||
session.mount("https://", adapter)
|
||||
response = session.get(url)
|
||||
print(url)
|
||||
if response.ok:
|
||||
open(out_path, "wb").write(response.content)
|
||||
|
||||
|
||||
@ -29,6 +29,9 @@ from tee_popen import TeePopen
|
||||
def get_run_command(
|
||||
build_path, result_folder, repo_tests_path, server_log_folder, image
|
||||
):
|
||||
download_previous_release = os.path.join(
|
||||
repo_tests_path, "ci/download_previous_release.py"
|
||||
)
|
||||
cmd = (
|
||||
"docker run --cap-add=SYS_PTRACE "
|
||||
"-e S3_URL='https://clickhouse-datasets.s3.amazonaws.com' "
|
||||
@ -36,6 +39,7 @@ def get_run_command(
|
||||
f"--volume={result_folder}:/test_output "
|
||||
f"--volume={repo_tests_path}:/usr/share/clickhouse-test "
|
||||
f"--volume={server_log_folder}:/var/log/clickhouse-server {image}"
|
||||
f"--volume={download_previous_release}:/usr/local/bin/download_previous_release "
|
||||
)
|
||||
|
||||
return cmd
|
||||
|
||||
Loading…
Reference in New Issue
Block a user