From e76ebdfb475401fd057f365f9e34cbb6df03da0a Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 29 Jun 2022 12:33:20 +0200 Subject: [PATCH 1/4] Change clickhouse-{client,server} packages architecture --- docs/_includes/install/tgz.sh | 30 +++++-- docs/en/development/continuous-integration.md | 6 +- docs/en/getting-started/install.md | 30 +++++-- docs/ru/getting-started/install.md | 30 +++++-- docs/zh/development/continuous-integration.md | 7 +- docs/zh/getting-started/install.md | 30 +++++-- packages/clickhouse-client.yaml | 2 +- packages/clickhouse-server.yaml | 2 +- tests/ci/download_previous_release.py | 86 ++++++++----------- 9 files changed, 131 insertions(+), 92 deletions(-) diff --git a/docs/_includes/install/tgz.sh b/docs/_includes/install/tgz.sh index 4ba5890b32b..d6d7cd8bc36 100644 --- a/docs/_includes/install/tgz.sh +++ b/docs/_includes/install/tgz.sh @@ -1,20 +1,34 @@ LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz" -tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" +case $(uname -m) in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) echo "Unknown architecture $(uname -m)"; exit 1 ;; +esac + +for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client +do + curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \ + || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz" +done + +exit 0 + +tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" sudo /etc/init.d/clickhouse-server start -tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh" diff --git a/docs/en/development/continuous-integration.md b/docs/en/development/continuous-integration.md index f8fcead3ca2..7b2da4416d6 100644 --- a/docs/en/development/continuous-integration.md +++ b/docs/en/development/continuous-integration.md @@ -123,12 +123,10 @@ Builds ClickHouse in various configurations for use in further steps. You have t - **Build log**: link to the building and files copying log, useful when build failed. - **Build time**. - **Artifacts**: build result files (with `XXX` being the server version e.g. `20.8.1.4344`). - - `clickhouse-client_XXX_all.deb` + - `clickhouse-client_XXX_amd64.deb` - `clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb` - `clickhouse-common-staticXXX_amd64.deb` - - `clickhouse-server_XXX_all.deb` - - `clickhouse_XXX_amd64.buildinfo` - - `clickhouse_XXX_amd64.changes` + - `clickhouse-server_XXX_amd64.deb` - `clickhouse`: Main built binary. - `clickhouse-odbc-bridge` - `unit_tests_dbms`: GoogleTest binary with ClickHouse unit tests. diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 8505b040fa3..a5e6495d8d8 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -127,22 +127,36 @@ After that downloaded archives should be unpacked and installed with installatio LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION-amd64.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION-amd64.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION-amd64.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION-amd64.tgz" -tar -xzvf "clickhouse-common-static-$LATEST_VERSION-amd64.tgz" +case $(uname -m) in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) echo "Unknown architecture $(uname -m)"; exit 1 ;; +esac + +for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client +do + curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \ + || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz" +done + +exit 0 + +tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-amd64.tgz" +tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-server-$LATEST_VERSION-amd64.tgz" +tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" sudo /etc/init.d/clickhouse-server start -tar -xzvf "clickhouse-client-$LATEST_VERSION-amd64.tgz" +tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh" ``` diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md index 09e91530473..9f3eabc73ae 100644 --- a/docs/ru/getting-started/install.md +++ b/docs/ru/getting-started/install.md @@ -124,22 +124,36 @@ sudo yum install clickhouse-server clickhouse-client LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz" -tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" +case $(uname -m) in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) echo "Unknown architecture $(uname -m)"; exit 1 ;; +esac + +for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client +do + curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \ + || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz" +done + +exit 0 + +tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" sudo /etc/init.d/clickhouse-server start -tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh" ``` diff --git a/docs/zh/development/continuous-integration.md b/docs/zh/development/continuous-integration.md index 5bebb3aec2a..836c4a35433 100644 --- a/docs/zh/development/continuous-integration.md +++ b/docs/zh/development/continuous-integration.md @@ -86,13 +86,10 @@ git push - **Build log**: 链接到构建和文件复制日志, 当构建失败时很有用. - **Build time**. - **Artifacts**: 构建结果文件 (`XXX`是服务器版本, 比如`20.8.1.4344`). - - `clickhouse-client_XXX_all.deb` + - `clickhouse-client_XXX_amd64.deb` -` clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb` - `clickhouse-common-staticXXX_amd64.deb` - - `clickhouse-server_XXX_all.deb` - - `clickhouse-test_XXX_all.deb` - - `clickhouse_XXX_amd64.buildinfo` - - `clickhouse_XXX_amd64.changes` + - `clickhouse-server_XXX_amd64.deb` - `clickhouse`: Main built binary. - `clickhouse-odbc-bridge` - `unit_tests_dbms`: 带有 ClickHouse 单元测试的 GoogleTest 二进制文件. diff --git a/docs/zh/getting-started/install.md b/docs/zh/getting-started/install.md index a8b803547a8..6a0b47607f5 100644 --- a/docs/zh/getting-started/install.md +++ b/docs/zh/getting-started/install.md @@ -121,22 +121,36 @@ sudo yum install clickhouse-server clickhouse-client LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz" -tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" +case $(uname -m) in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) echo "Unknown architecture $(uname -m)"; exit 1 ;; +esac + +for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client +do + curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \ + || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz" +done + +exit 0 + +tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" sudo /etc/init.d/clickhouse-server start -tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh" ``` diff --git a/packages/clickhouse-client.yaml b/packages/clickhouse-client.yaml index 5e53090b581..642d66f5475 100644 --- a/packages/clickhouse-client.yaml +++ b/packages/clickhouse-client.yaml @@ -1,7 +1,7 @@ # package sources should be placed in ${PWD}/root # nfpm should run from the same directory with a config name: "clickhouse-client" -arch: "all" +arch: "${DEB_ARCH}" # amd64, arm64 platform: "linux" version: "${CLICKHOUSE_VERSION_STRING}" vendor: "ClickHouse Inc." diff --git a/packages/clickhouse-server.yaml b/packages/clickhouse-server.yaml index ed56eb27e54..28995689754 100644 --- a/packages/clickhouse-server.yaml +++ b/packages/clickhouse-server.yaml @@ -1,7 +1,7 @@ # package sources should be placed in ${PWD}/root # nfpm should run from the same directory with a config name: "clickhouse-server" -arch: "all" +arch: "${DEB_ARCH}" # amd64, arm64 platform: "linux" version: "${CLICKHOUSE_VERSION_STRING}" vendor: "ClickHouse Inc." diff --git a/tests/ci/download_previous_release.py b/tests/ci/download_previous_release.py index fa03d164f23..86beed35b5a 100755 --- a/tests/ci/download_previous_release.py +++ b/tests/ci/download_previous_release.py @@ -4,25 +4,24 @@ import re import os import logging -import requests +import requests # type: ignore -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry +from requests.adapters import HTTPAdapter # type: ignore +from urllib3.util.retry import Retry # type: ignore CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags" -CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb" -CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static-dbg_{version}_amd64.deb" -CLICKHOUSE_SERVER_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-server_{version}_all.deb" -CLICKHOUSE_CLIENT_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-client_{version}_all.deb" - - +DOWNLOAD_PREFIX = ( + "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/" +) CLICKHOUSE_COMMON_STATIC_PACKET_NAME = "clickhouse-common-static_{version}_amd64.deb" CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME = ( "clickhouse-common-static-dbg_{version}_amd64.deb" ) -CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_all.deb" -CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_all.deb" +CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_amd64.deb" +CLICKHOUSE_SERVER_PACKET_FALLBACK = "clickhouse-server_{version}_all.deb" +CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_amd64.deb" +CLICKHOUSE_CLIENT_PACKET_FALLBACK = "clickhouse-client_{version}_all.deb" PACKETS_DIR = "previous_release_package_folder/" VERSION_PATTERN = r"((?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)" @@ -102,9 +101,10 @@ def download_packet(url, out_path, retries=10, backoff_factor=0.3): session.mount("http://", adapter) session.mount("https://", adapter) response = session.get(url) - print(url) - if response.ok: - open(out_path, "wb").write(response.content) + response.raise_for_status() + print(f"Download {url} to {out_path}") + with open(out_path, "wb") as fd: + fd.write(response.content) def download_packets(release, dest_path=PACKETS_DIR): @@ -113,43 +113,31 @@ def download_packets(release, dest_path=PACKETS_DIR): logging.info("Will download %s", release) - download_packet( - CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format( - version=release.version, type=release.type - ), - out_path=os.path.join( - dest_path, - CLICKHOUSE_COMMON_STATIC_PACKET_NAME.format(version=release.version), - ), - ) + def get_dest_path(pkg_name): + return os.path.join(dest_path, pkg_name) - download_packet( - CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL.format( - version=release.version, type=release.type - ), - out_path=os.path.join( - dest_path, - CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME.format(version=release.version), - ), - ) + for pkg in ( + CLICKHOUSE_COMMON_STATIC_PACKET_NAME, + CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME, + ): + url = (DOWNLOAD_PREFIX + pkg).format(version=release.version, type=release.type) + pkg_name = get_dest_path(pkg.format(version=release.version)) + download_packet(url, pkg_name) - download_packet( - CLICKHOUSE_SERVER_DOWNLOAD_URL.format( - version=release.version, type=release.type - ), - out_path=os.path.join( - dest_path, CLICKHOUSE_SERVER_PACKET_NAME.format(version=release.version) - ), - ) - - download_packet( - CLICKHOUSE_CLIENT_DOWNLOAD_URL.format( - version=release.version, type=release.type - ), - out_path=os.path.join( - dest_path, CLICKHOUSE_CLIENT_PACKET_NAME.format(version=release.version) - ), - ) + for pkg, fallback in ( + (CLICKHOUSE_SERVER_PACKET_NAME, CLICKHOUSE_SERVER_PACKET_FALLBACK), + (CLICKHOUSE_CLIENT_PACKET_NAME, CLICKHOUSE_CLIENT_PACKET_FALLBACK), + ): + url = (DOWNLOAD_PREFIX + pkg).format(version=release.version, type=release.type) + pkg_name = get_dest_path(pkg.format(version=release.version)) + try: + download_packet(url, pkg_name) + except Exception: + url = (DOWNLOAD_PREFIX + fallback).format( + version=release.version, type=release.type + ) + pkg_name = get_dest_path(fallback.format(version=release.version)) + download_packet(url, pkg_name) def download_previous_release(dest_path): From 6925bd6a03c3f8822681c1aa8c4651cd3bca5eff Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 30 Jun 2022 11:58:24 +0200 Subject: [PATCH 2/4] Fix hardcoded retries for get_with_retries --- tests/ci/build_download_helper.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index 67e1c6ee85d..f5eb72dddee 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -20,15 +20,17 @@ def get_with_retries( sleep: int = 3, **kwargs, ) -> requests.Response: - logging.info("Getting URL with %i and sleep %i in between: %s", retries, sleep, url) + logging.info( + "Getting URL with %i tries and sleep %i in between: %s", retries, sleep, url + ) exc = None # type: Optional[Exception] - for i in range(DOWNLOAD_RETRIES_COUNT): + for i in range(retries): try: response = requests.get(url, **kwargs) response.raise_for_status() break except Exception as e: - if i + 1 < DOWNLOAD_RETRIES_COUNT: + if i + 1 < retries: logging.info("Exception '%s' while getting, retry %i", e, i + 1) time.sleep(sleep) From 738769d1f7ef93a905a2b52e4adcc903a3c82178 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 30 Jun 2022 15:41:50 +0200 Subject: [PATCH 3/4] Download arm64 packages and push to artifactory --- tests/ci/push_to_artifactory.py | 139 +++++++++++++++++++++----------- 1 file changed, 91 insertions(+), 48 deletions(-) diff --git a/tests/ci/push_to_artifactory.py b/tests/ci/push_to_artifactory.py index be977bdd907..98de315ddae 100755 --- a/tests/ci/push_to_artifactory.py +++ b/tests/ci/push_to_artifactory.py @@ -4,11 +4,12 @@ import argparse import logging import os import re -from typing import List, Tuple +from collections import namedtuple +from typing import Dict, List, Tuple from artifactory import ArtifactorySaaSPath # type: ignore from build_download_helper import dowload_build_with_progress -from env_helper import RUNNER_TEMP +from env_helper import RUNNER_TEMP, S3_BUILDS_BUCKET from git_helper import TAG_REGEXP, commit, removeprefix, removesuffix @@ -25,88 +26,144 @@ TEMP_PATH = os.path.join(RUNNER_TEMP, "push_to_artifactory") JFROG_API_KEY = getenv("JFROG_API_KEY", "") JFROG_TOKEN = getenv("JFROG_TOKEN", "") +CheckDesc = namedtuple("CheckDesc", ("check_name", "deb_arch", "rpm_arch")) + class Packages: - rpm_arch = dict(all="noarch", amd64="x86_64") + checks = ( + CheckDesc("package_release", "amd64", "x86_64"), + CheckDesc("package_aarch64", "arm64", "aarch64"), + ) packages = ( - ("clickhouse-client", "all"), - ("clickhouse-common-static", "amd64"), - ("clickhouse-common-static-dbg", "amd64"), - ("clickhouse-server", "all"), + "clickhouse-client", + "clickhouse-common-static", + "clickhouse-common-static-dbg", + "clickhouse-server", ) def __init__(self, version: str): - self.deb = tuple( - "_".join((name, version, arch + ".deb")) for name, arch in self.packages - ) + # Dicts of name: s3_path_suffix + self.deb = {} # type: Dict[str, str] + self.rpm = {} # type: Dict[str, str] + self.tgz = {} # type: Dict[str, str] + for check in self.checks: + for name in self.packages: + deb = f"{name}_{version}_{check.deb_arch}.deb" + self.deb[deb] = f"{check.check_name}/{deb}" - self.rpm = tuple( - "-".join((name, version + "." + self.rpm_arch[arch] + ".rpm")) - for name, arch in self.packages - ) + rpm = f"{name}-{version}.{check.rpm_arch}.rpm" + self.rpm[rpm] = f"{check.check_name}/{rpm}" - self.tgz = tuple(f"{name}-{version}-amd64.tgz" for name, _ in self.packages) + tgz = f"{name}-{version}-{check.deb_arch}.tgz" + self.tgz[tgz] = f"{check.check_name}/{tgz}" def arch(self, deb_pkg: str) -> str: if deb_pkg not in self.deb: raise ValueError(f"{deb_pkg} not in {self.deb}") return removesuffix(deb_pkg, ".deb").split("_")[-1] + def replace_with_fallback(self, name: str): + if name.endswith(".deb"): + suffix = self.deb.pop(name) + self.deb[self.fallback_to_all(name)] = self.fallback_to_all(suffix) + elif name.endswith(".rpm"): + suffix = self.rpm.pop(name) + self.rpm[self.fallback_to_all(name)] = self.fallback_to_all(suffix) + elif name.endswith(".tgz"): + suffix = self.tgz.pop(name) + self.tgz[self.fallback_to_all(name)] = self.fallback_to_all(suffix) + else: + raise KeyError(f"unknown package type for {name}") + @staticmethod def path(package_file: str) -> str: return os.path.join(TEMP_PATH, package_file) + @staticmethod + def fallback_to_all(url_or_name: str): + """Until July 2022 we had clickhouse-server and clickhouse-client with + arch 'all'""" + # deb + if url_or_name.endswith("amd64.deb") or url_or_name.endswith("arm64.deb"): + return f"{url_or_name[:-9]}all.deb" + # rpm + if url_or_name.endswith("x86_64.rpm") or url_or_name.endswith("aarch64.rpm"): + new = removesuffix(removesuffix(url_or_name, "x86_64.rpm"), "aarch64.rpm") + return f"{new}noarch.rpm" + # tgz + if url_or_name.endswith("-amd64.tgz") or url_or_name.endswith("-arm64.tgz"): + return f"{url_or_name[:-10]}.tgz" + return url_or_name + class S3: template = ( "https://s3.amazonaws.com/" # "clickhouse-builds/" - "{bucket_name}/" + f"{S3_BUILDS_BUCKET}/" # "33333/" or "21.11/" from --release, if pull request is omitted "{pr}/" # "2bef313f75e4cacc6ea2ef2133e8849ecf0385ec/" "{commit}/" - # "package_release/" - "{check_name}/" - # "clickhouse-common-static_21.11.5.0_amd64.deb" - "{package}" + # "package_release/clickhouse-common-static_21.11.5.0_amd64.deb" + "{s3_path_suffix}" ) def __init__( self, - bucket_name: str, pr: int, commit: str, - check_name: str, version: str, force_download: bool, ): self._common = dict( - bucket_name=bucket_name, pr=pr, commit=commit, - check_name=check_name, ) self.force_download = force_download self.packages = Packages(version) - def download_package(self, package_file: str): - if not self.force_download and os.path.exists(Packages.path(package_file)): + def download_package(self, package_file: str, s3_path_suffix: str): + path = Packages.path(package_file) + fallback_path = Packages.fallback_to_all(path) + if not self.force_download and ( + os.path.exists(path) or os.path.exists(fallback_path) + ): + if os.path.exists(fallback_path): + self.packages.replace_with_fallback(package_file) + return - url = self.template.format_map({**self._common, "package": package_file}) - dowload_build_with_progress(url, Packages.path(package_file)) + url = self.template.format_map( + {**self._common, "s3_path_suffix": s3_path_suffix} + ) + try: + dowload_build_with_progress(url, path) + except Exception as e: + if "Cannot download dataset from" in e.args[0]: + new_url = Packages.fallback_to_all(url) + logging.warning( + "Fallback downloading %s for old release", fallback_path + ) + dowload_build_with_progress(new_url, fallback_path) + self.packages.replace_with_fallback(package_file) def download_deb(self): - for package_file in self.packages.deb: - self.download_package(package_file) + # Copy to have a way to pop/add fallback packages + packages = self.packages.deb.copy() + for package_file, s3_path_suffix in packages.items(): + self.download_package(package_file, s3_path_suffix) def download_rpm(self): - for package_file in self.packages.rpm: - self.download_package(package_file) + # Copy to have a way to pop/add fallback packages + packages = self.packages.rpm.copy() + for package_file, s3_path_suffix in packages.items(): + self.download_package(package_file, s3_path_suffix) def download_tgz(self): - for package_file in self.packages.tgz: - self.download_package(package_file) + # Copy to have a way to pop/add fallback packages + packages = self.packages.tgz.copy() + for package_file, s3_path_suffix in packages.items(): + self.download_package(package_file, s3_path_suffix) class Release: @@ -223,17 +280,6 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--commit", required=True, type=commit, help="commit hash for S3 bucket" ) - parser.add_argument( - "--bucket-name", - default="clickhouse-builds", - help="AWS S3 bucket name", - ) - parser.add_argument( - "--check-name", - default="package_release", - help="check name, a part of bucket path, " - "will be converted to lower case with spaces->underscore", - ) parser.add_argument( "--all", action="store_true", help="implies all deb, rpm and tgz" ) @@ -276,7 +322,6 @@ def parse_args() -> argparse.Namespace: args.deb = args.rpm = args.tgz = True if not (args.deb or args.rpm or args.tgz): parser.error("at least one of --deb, --rpm or --tgz should be specified") - args.check_name = args.check_name.lower().replace(" ", "_") if args.pull_request == 0: args.pull_request = ".".join(args.release.version_parts[:2]) return args @@ -305,10 +350,8 @@ def main(): args = parse_args() os.makedirs(TEMP_PATH, exist_ok=True) s3 = S3( - args.bucket_name, args.pull_request, args.commit, - args.check_name, args.release.version, args.force_download, ) From aeaf16ae2ff5c02c8967723fc9bce310e3dc869e Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 30 Jun 2022 18:01:17 +0200 Subject: [PATCH 4/4] Use the script push_to_artifactory.py always from master --- .github/workflows/release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 83e22c0b7c6..4d57ae450c4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,6 +21,9 @@ jobs: EOF - name: Check out repository code uses: actions/checkout@v2 + with: + # Always use the most recent script version + ref: master - name: Download packages and push to Artifactory run: | rm -rf "$TEMP_PATH" && mkdir -p "$TEMP_PATH"