From e76ebdfb475401fd057f365f9e34cbb6df03da0a Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 29 Jun 2022 12:33:20 +0200 Subject: [PATCH] Change clickhouse-{client,server} packages architecture --- docs/_includes/install/tgz.sh | 30 +++++-- docs/en/development/continuous-integration.md | 6 +- docs/en/getting-started/install.md | 30 +++++-- docs/ru/getting-started/install.md | 30 +++++-- docs/zh/development/continuous-integration.md | 7 +- docs/zh/getting-started/install.md | 30 +++++-- packages/clickhouse-client.yaml | 2 +- packages/clickhouse-server.yaml | 2 +- tests/ci/download_previous_release.py | 86 ++++++++----------- 9 files changed, 131 insertions(+), 92 deletions(-) diff --git a/docs/_includes/install/tgz.sh b/docs/_includes/install/tgz.sh index 4ba5890b32b..d6d7cd8bc36 100644 --- a/docs/_includes/install/tgz.sh +++ b/docs/_includes/install/tgz.sh @@ -1,20 +1,34 @@ LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz" -tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" +case $(uname -m) in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) echo "Unknown architecture $(uname -m)"; exit 1 ;; +esac + +for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client +do + curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \ + || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz" +done + +exit 0 + +tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" sudo /etc/init.d/clickhouse-server start -tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh" diff --git a/docs/en/development/continuous-integration.md b/docs/en/development/continuous-integration.md index f8fcead3ca2..7b2da4416d6 100644 --- a/docs/en/development/continuous-integration.md +++ b/docs/en/development/continuous-integration.md @@ -123,12 +123,10 @@ Builds ClickHouse in various configurations for use in further steps. You have t - **Build log**: link to the building and files copying log, useful when build failed. - **Build time**. - **Artifacts**: build result files (with `XXX` being the server version e.g. `20.8.1.4344`). - - `clickhouse-client_XXX_all.deb` + - `clickhouse-client_XXX_amd64.deb` - `clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb` - `clickhouse-common-staticXXX_amd64.deb` - - `clickhouse-server_XXX_all.deb` - - `clickhouse_XXX_amd64.buildinfo` - - `clickhouse_XXX_amd64.changes` + - `clickhouse-server_XXX_amd64.deb` - `clickhouse`: Main built binary. - `clickhouse-odbc-bridge` - `unit_tests_dbms`: GoogleTest binary with ClickHouse unit tests. diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 8505b040fa3..a5e6495d8d8 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -127,22 +127,36 @@ After that downloaded archives should be unpacked and installed with installatio LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION-amd64.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION-amd64.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION-amd64.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION-amd64.tgz" -tar -xzvf "clickhouse-common-static-$LATEST_VERSION-amd64.tgz" +case $(uname -m) in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) echo "Unknown architecture $(uname -m)"; exit 1 ;; +esac + +for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client +do + curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \ + || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz" +done + +exit 0 + +tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-amd64.tgz" +tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-server-$LATEST_VERSION-amd64.tgz" +tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" sudo /etc/init.d/clickhouse-server start -tar -xzvf "clickhouse-client-$LATEST_VERSION-amd64.tgz" +tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh" ``` diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md index 09e91530473..9f3eabc73ae 100644 --- a/docs/ru/getting-started/install.md +++ b/docs/ru/getting-started/install.md @@ -124,22 +124,36 @@ sudo yum install clickhouse-server clickhouse-client LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz" -tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" +case $(uname -m) in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) echo "Unknown architecture $(uname -m)"; exit 1 ;; +esac + +for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client +do + curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \ + || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz" +done + +exit 0 + +tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" sudo /etc/init.d/clickhouse-server start -tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh" ``` diff --git a/docs/zh/development/continuous-integration.md b/docs/zh/development/continuous-integration.md index 5bebb3aec2a..836c4a35433 100644 --- a/docs/zh/development/continuous-integration.md +++ b/docs/zh/development/continuous-integration.md @@ -86,13 +86,10 @@ git push - **Build log**: 链接到构建和文件复制日志, 当构建失败时很有用. - **Build time**. - **Artifacts**: 构建结果文件 (`XXX`是服务器版本, 比如`20.8.1.4344`). - - `clickhouse-client_XXX_all.deb` + - `clickhouse-client_XXX_amd64.deb` -` clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb` - `clickhouse-common-staticXXX_amd64.deb` - - `clickhouse-server_XXX_all.deb` - - `clickhouse-test_XXX_all.deb` - - `clickhouse_XXX_amd64.buildinfo` - - `clickhouse_XXX_amd64.changes` + - `clickhouse-server_XXX_amd64.deb` - `clickhouse`: Main built binary. - `clickhouse-odbc-bridge` - `unit_tests_dbms`: 带有 ClickHouse 单元测试的 GoogleTest 二进制文件. diff --git a/docs/zh/getting-started/install.md b/docs/zh/getting-started/install.md index a8b803547a8..6a0b47607f5 100644 --- a/docs/zh/getting-started/install.md +++ b/docs/zh/getting-started/install.md @@ -121,22 +121,36 @@ sudo yum install clickhouse-server clickhouse-client LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz" -tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" +case $(uname -m) in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) echo "Unknown architecture $(uname -m)"; exit 1 ;; +esac + +for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client +do + curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \ + || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz" +done + +exit 0 + +tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" sudo /etc/init.d/clickhouse-server start -tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh" ``` diff --git a/packages/clickhouse-client.yaml b/packages/clickhouse-client.yaml index 5e53090b581..642d66f5475 100644 --- a/packages/clickhouse-client.yaml +++ b/packages/clickhouse-client.yaml @@ -1,7 +1,7 @@ # package sources should be placed in ${PWD}/root # nfpm should run from the same directory with a config name: "clickhouse-client" -arch: "all" +arch: "${DEB_ARCH}" # amd64, arm64 platform: "linux" version: "${CLICKHOUSE_VERSION_STRING}" vendor: "ClickHouse Inc." diff --git a/packages/clickhouse-server.yaml b/packages/clickhouse-server.yaml index ed56eb27e54..28995689754 100644 --- a/packages/clickhouse-server.yaml +++ b/packages/clickhouse-server.yaml @@ -1,7 +1,7 @@ # package sources should be placed in ${PWD}/root # nfpm should run from the same directory with a config name: "clickhouse-server" -arch: "all" +arch: "${DEB_ARCH}" # amd64, arm64 platform: "linux" version: "${CLICKHOUSE_VERSION_STRING}" vendor: "ClickHouse Inc." diff --git a/tests/ci/download_previous_release.py b/tests/ci/download_previous_release.py index fa03d164f23..86beed35b5a 100755 --- a/tests/ci/download_previous_release.py +++ b/tests/ci/download_previous_release.py @@ -4,25 +4,24 @@ import re import os import logging -import requests +import requests # type: ignore -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry +from requests.adapters import HTTPAdapter # type: ignore +from urllib3.util.retry import Retry # type: ignore CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags" -CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb" -CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static-dbg_{version}_amd64.deb" -CLICKHOUSE_SERVER_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-server_{version}_all.deb" -CLICKHOUSE_CLIENT_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-client_{version}_all.deb" - - +DOWNLOAD_PREFIX = ( + "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/" +) CLICKHOUSE_COMMON_STATIC_PACKET_NAME = "clickhouse-common-static_{version}_amd64.deb" CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME = ( "clickhouse-common-static-dbg_{version}_amd64.deb" ) -CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_all.deb" -CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_all.deb" +CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_amd64.deb" +CLICKHOUSE_SERVER_PACKET_FALLBACK = "clickhouse-server_{version}_all.deb" +CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_amd64.deb" +CLICKHOUSE_CLIENT_PACKET_FALLBACK = "clickhouse-client_{version}_all.deb" PACKETS_DIR = "previous_release_package_folder/" VERSION_PATTERN = r"((?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)" @@ -102,9 +101,10 @@ def download_packet(url, out_path, retries=10, backoff_factor=0.3): session.mount("http://", adapter) session.mount("https://", adapter) response = session.get(url) - print(url) - if response.ok: - open(out_path, "wb").write(response.content) + response.raise_for_status() + print(f"Download {url} to {out_path}") + with open(out_path, "wb") as fd: + fd.write(response.content) def download_packets(release, dest_path=PACKETS_DIR): @@ -113,43 +113,31 @@ def download_packets(release, dest_path=PACKETS_DIR): logging.info("Will download %s", release) - download_packet( - CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format( - version=release.version, type=release.type - ), - out_path=os.path.join( - dest_path, - CLICKHOUSE_COMMON_STATIC_PACKET_NAME.format(version=release.version), - ), - ) + def get_dest_path(pkg_name): + return os.path.join(dest_path, pkg_name) - download_packet( - CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL.format( - version=release.version, type=release.type - ), - out_path=os.path.join( - dest_path, - CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME.format(version=release.version), - ), - ) + for pkg in ( + CLICKHOUSE_COMMON_STATIC_PACKET_NAME, + CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME, + ): + url = (DOWNLOAD_PREFIX + pkg).format(version=release.version, type=release.type) + pkg_name = get_dest_path(pkg.format(version=release.version)) + download_packet(url, pkg_name) - download_packet( - CLICKHOUSE_SERVER_DOWNLOAD_URL.format( - version=release.version, type=release.type - ), - out_path=os.path.join( - dest_path, CLICKHOUSE_SERVER_PACKET_NAME.format(version=release.version) - ), - ) - - download_packet( - CLICKHOUSE_CLIENT_DOWNLOAD_URL.format( - version=release.version, type=release.type - ), - out_path=os.path.join( - dest_path, CLICKHOUSE_CLIENT_PACKET_NAME.format(version=release.version) - ), - ) + for pkg, fallback in ( + (CLICKHOUSE_SERVER_PACKET_NAME, CLICKHOUSE_SERVER_PACKET_FALLBACK), + (CLICKHOUSE_CLIENT_PACKET_NAME, CLICKHOUSE_CLIENT_PACKET_FALLBACK), + ): + url = (DOWNLOAD_PREFIX + pkg).format(version=release.version, type=release.type) + pkg_name = get_dest_path(pkg.format(version=release.version)) + try: + download_packet(url, pkg_name) + except Exception: + url = (DOWNLOAD_PREFIX + fallback).format( + version=release.version, type=release.type + ) + pkg_name = get_dest_path(fallback.format(version=release.version)) + download_packet(url, pkg_name) def download_previous_release(dest_path):