thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #38580 from ClickHouse/artifactory-arm64

Browse Source 
Push arm64 packages to artifactory and release assets
This commit is contained in:
Mikhail f. Shiryaev 2022-07-04 12:31:52 +02:00 committed by GitHub
commit 7c812df47e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 230 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/workflows/release.yml vendored
View File

@ -21,6 +21,9 @@ jobs:
EOF
- name: Check out repository code
uses: actions/checkout@v2
with:
# Always use the most recent script version
ref: master
- name: Download packages and push to Artifactory
run: |
rm -rf "$TEMP_PATH" && mkdir -p "$TEMP_PATH"

30
docs/_includes/install/tgz.sh
View File

@ -1,20 +1,34 @@
LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
export LATEST_VERSION
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz"
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz"
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz"
tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
case $(uname -m) in
x86_64) ARCH=amd64 ;;
aarch64) ARCH=arm64 ;;
*) echo "Unknown architecture $(uname -m)"; exit 1 ;;
esac
for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client
do
curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \
|| curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz"
done
exit 0
tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh"
tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh"
tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh"
sudo /etc/init.d/clickhouse-server start
tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh"

6
docs/en/development/continuous-integration.md
View File

@ -123,12 +123,10 @@ Builds ClickHouse in various configurations for use in further steps. You have t
- **Build log**: link to the building and files copying log, useful when build failed.
- **Build time**.
- **Artifacts**: build result files (with `XXX` being the server version e.g. `20.8.1.4344`).
- `clickhouse-client_XXX_all.deb`
- `clickhouse-client_XXX_amd64.deb`
- `clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb`
- `clickhouse-common-staticXXX_amd64.deb`
- `clickhouse-server_XXX_all.deb`
- `clickhouse_XXX_amd64.buildinfo`
- `clickhouse_XXX_amd64.changes`
- `clickhouse-server_XXX_amd64.deb`
- `clickhouse`: Main built binary.
- `clickhouse-odbc-bridge`
- `unit_tests_dbms`: GoogleTest binary with ClickHouse unit tests.

30
docs/en/getting-started/install.md
View File

@ -127,22 +127,36 @@ After that downloaded archives should be unpacked and installed with installatio
LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
export LATEST_VERSION
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION-amd64.tgz"
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION-amd64.tgz"
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION-amd64.tgz"
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION-amd64.tgz"
tar -xzvf "clickhouse-common-static-$LATEST_VERSION-amd64.tgz"
case $(uname -m) in
x86_64) ARCH=amd64 ;;
aarch64) ARCH=arm64 ;;
*) echo "Unknown architecture $(uname -m)"; exit 1 ;;
esac
for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client
do
curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \
|| curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz"
done
exit 0
tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh"
tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-amd64.tgz"
tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh"
tar -xzvf "clickhouse-server-$LATEST_VERSION-amd64.tgz"
tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh"
sudo /etc/init.d/clickhouse-server start
tar -xzvf "clickhouse-client-$LATEST_VERSION-amd64.tgz"
tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh"
```

30
docs/ru/getting-started/install.md
View File

@ -124,22 +124,36 @@ sudo yum install clickhouse-server clickhouse-client
LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
export LATEST_VERSION
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz"
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz"
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz"
tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
case $(uname -m) in
x86_64) ARCH=amd64 ;;
aarch64) ARCH=arm64 ;;
*) echo "Unknown architecture $(uname -m)"; exit 1 ;;
esac
for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client
do
curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \
|| curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz"
done
exit 0
tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh"
tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh"
tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh"
sudo /etc/init.d/clickhouse-server start
tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh"
```

7
docs/zh/development/continuous-integration.md
View File

@ -86,13 +86,10 @@ git push
- **Build log**: 链接到构建和文件复制日志, 当构建失败时很有用.
- **Build time**.
- **Artifacts**: 构建结果文件 (`XXX`是服务器版本, 比如`20.8.1.4344`).
- `clickhouse-client_XXX_all.deb`
- `clickhouse-client_XXX_amd64.deb`
-` clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb`
- `clickhouse-common-staticXXX_amd64.deb`
- `clickhouse-server_XXX_all.deb`
- `clickhouse-test_XXX_all.deb`
- `clickhouse_XXX_amd64.buildinfo`
- `clickhouse_XXX_amd64.changes`
- `clickhouse-server_XXX_amd64.deb`
- `clickhouse`: Main built binary.
- `clickhouse-odbc-bridge`
- `unit_tests_dbms`: 带有 ClickHouse 单元测试的 GoogleTest 二进制文件.

30
docs/zh/getting-started/install.md
View File

@ -121,22 +121,36 @@ sudo yum install clickhouse-server clickhouse-client
LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
export LATEST_VERSION
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz"
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz"
curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz"
tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
case $(uname -m) in
x86_64) ARCH=amd64 ;;
aarch64) ARCH=arm64 ;;
*) echo "Unknown architecture $(uname -m)"; exit 1 ;;
esac
for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client
do
curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \
|| curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz"
done
exit 0
tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh"
tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh"
tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh"
sudo /etc/init.d/clickhouse-server start
tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh"
```

2
packages/clickhouse-client.yaml
View File

@ -1,7 +1,7 @@
# package sources should be placed in ${PWD}/root
# nfpm should run from the same directory with a config
name: "clickhouse-client"
arch: "all"
arch: "${DEB_ARCH}" # amd64, arm64
platform: "linux"
version: "${CLICKHOUSE_VERSION_STRING}"
vendor: "ClickHouse Inc."

2
packages/clickhouse-server.yaml
View File

@ -1,7 +1,7 @@
# package sources should be placed in ${PWD}/root
# nfpm should run from the same directory with a config
name: "clickhouse-server"
arch: "all"
arch: "${DEB_ARCH}" # amd64, arm64
platform: "linux"
version: "${CLICKHOUSE_VERSION_STRING}"
vendor: "ClickHouse Inc."

8
tests/ci/build_download_helper.py
View File

@ -20,15 +20,17 @@ def get_with_retries(
sleep: int = 3,
**kwargs,
) -> requests.Response:
logging.info("Getting URL with %i and sleep %i in between: %s", retries, sleep, url)
logging.info(
"Getting URL with %i tries and sleep %i in between: %s", retries, sleep, url
)
exc = None # type: Optional[Exception]
for i in range(DOWNLOAD_RETRIES_COUNT):
for i in range(retries):
try:
response = requests.get(url, **kwargs)
response.raise_for_status()
break
except Exception as e:
if i + 1 < DOWNLOAD_RETRIES_COUNT:
if i + 1 < retries:
logging.info("Exception '%s' while getting, retry %i", e, i + 1)
time.sleep(sleep)

86
tests/ci/download_previous_release.py
View File

@ -4,25 +4,24 @@ import re
import os
import logging
import requests
import requests # type: ignore
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter # type: ignore
from urllib3.util.retry import Retry # type: ignore
CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags"
CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb"
CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static-dbg_{version}_amd64.deb"
CLICKHOUSE_SERVER_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-server_{version}_all.deb"
CLICKHOUSE_CLIENT_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-client_{version}_all.deb"
DOWNLOAD_PREFIX = (
"https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/"
)
CLICKHOUSE_COMMON_STATIC_PACKET_NAME = "clickhouse-common-static_{version}_amd64.deb"
CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME = (
"clickhouse-common-static-dbg_{version}_amd64.deb"
)
CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_all.deb"
CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_all.deb"
CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_amd64.deb"
CLICKHOUSE_SERVER_PACKET_FALLBACK = "clickhouse-server_{version}_all.deb"
CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_amd64.deb"
CLICKHOUSE_CLIENT_PACKET_FALLBACK = "clickhouse-client_{version}_all.deb"
PACKETS_DIR = "previous_release_package_folder/"
VERSION_PATTERN = r"((?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)"
@ -102,9 +101,10 @@ def download_packet(url, out_path, retries=10, backoff_factor=0.3):
session.mount("http://", adapter)
session.mount("https://", adapter)
response = session.get(url)
print(url)
if response.ok:
open(out_path, "wb").write(response.content)
response.raise_for_status()
print(f"Download {url} to {out_path}")
with open(out_path, "wb") as fd:
fd.write(response.content)
def download_packets(release, dest_path=PACKETS_DIR):
@ -113,43 +113,31 @@ def download_packets(release, dest_path=PACKETS_DIR):
logging.info("Will download %s", release)
download_packet(
CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format(
version=release.version, type=release.type
),
out_path=os.path.join(
dest_path,
CLICKHOUSE_COMMON_STATIC_PACKET_NAME.format(version=release.version),
),
)
def get_dest_path(pkg_name):
return os.path.join(dest_path, pkg_name)
download_packet(
CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL.format(
version=release.version, type=release.type
),
out_path=os.path.join(
dest_path,
CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME.format(version=release.version),
),
)
for pkg in (
CLICKHOUSE_COMMON_STATIC_PACKET_NAME,
CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME,
):
url = (DOWNLOAD_PREFIX + pkg).format(version=release.version, type=release.type)
pkg_name = get_dest_path(pkg.format(version=release.version))
download_packet(url, pkg_name)
download_packet(
CLICKHOUSE_SERVER_DOWNLOAD_URL.format(
version=release.version, type=release.type
),
out_path=os.path.join(
dest_path, CLICKHOUSE_SERVER_PACKET_NAME.format(version=release.version)
),
)
download_packet(
CLICKHOUSE_CLIENT_DOWNLOAD_URL.format(
version=release.version, type=release.type
),
out_path=os.path.join(
dest_path, CLICKHOUSE_CLIENT_PACKET_NAME.format(version=release.version)
),
)
for pkg, fallback in (
(CLICKHOUSE_SERVER_PACKET_NAME, CLICKHOUSE_SERVER_PACKET_FALLBACK),
(CLICKHOUSE_CLIENT_PACKET_NAME, CLICKHOUSE_CLIENT_PACKET_FALLBACK),
):
url = (DOWNLOAD_PREFIX + pkg).format(version=release.version, type=release.type)
pkg_name = get_dest_path(pkg.format(version=release.version))
try:
download_packet(url, pkg_name)
except Exception:
url = (DOWNLOAD_PREFIX + fallback).format(
version=release.version, type=release.type
)
pkg_name = get_dest_path(fallback.format(version=release.version))
download_packet(url, pkg_name)
def download_previous_release(dest_path):

139
tests/ci/push_to_artifactory.py
View File

@ -4,11 +4,12 @@ import argparse
import logging
import os
import re
from typing import List, Tuple
from collections import namedtuple
from typing import Dict, List, Tuple
from artifactory import ArtifactorySaaSPath # type: ignore
from build_download_helper import dowload_build_with_progress
from env_helper import RUNNER_TEMP
from env_helper import RUNNER_TEMP, S3_BUILDS_BUCKET
from git_helper import TAG_REGEXP, commit, removeprefix, removesuffix
@ -25,88 +26,144 @@ TEMP_PATH = os.path.join(RUNNER_TEMP, "push_to_artifactory")
JFROG_API_KEY = getenv("JFROG_API_KEY", "")
JFROG_TOKEN = getenv("JFROG_TOKEN", "")
CheckDesc = namedtuple("CheckDesc", ("check_name", "deb_arch", "rpm_arch"))
class Packages:
rpm_arch = dict(all="noarch", amd64="x86_64")
checks = (
CheckDesc("package_release", "amd64", "x86_64"),
CheckDesc("package_aarch64", "arm64", "aarch64"),
)
packages = (
("clickhouse-client", "all"),
("clickhouse-common-static", "amd64"),
("clickhouse-common-static-dbg", "amd64"),
("clickhouse-server", "all"),
"clickhouse-client",
"clickhouse-common-static",
"clickhouse-common-static-dbg",
"clickhouse-server",
)
def __init__(self, version: str):
self.deb = tuple(
"_".join((name, version, arch + ".deb")) for name, arch in self.packages
)
# Dicts of name: s3_path_suffix
self.deb = {} # type: Dict[str, str]
self.rpm = {} # type: Dict[str, str]
self.tgz = {} # type: Dict[str, str]
for check in self.checks:
for name in self.packages:
deb = f"{name}_{version}_{check.deb_arch}.deb"
self.deb[deb] = f"{check.check_name}/{deb}"
self.rpm = tuple(
"-".join((name, version + "." + self.rpm_arch[arch] + ".rpm"))
for name, arch in self.packages
)
rpm = f"{name}-{version}.{check.rpm_arch}.rpm"
self.rpm[rpm] = f"{check.check_name}/{rpm}"
self.tgz = tuple(f"{name}-{version}-amd64.tgz" for name, _ in self.packages)
tgz = f"{name}-{version}-{check.deb_arch}.tgz"
self.tgz[tgz] = f"{check.check_name}/{tgz}"
def arch(self, deb_pkg: str) -> str:
if deb_pkg not in self.deb:
raise ValueError(f"{deb_pkg} not in {self.deb}")
return removesuffix(deb_pkg, ".deb").split("_")[-1]
def replace_with_fallback(self, name: str):
if name.endswith(".deb"):
suffix = self.deb.pop(name)
self.deb[self.fallback_to_all(name)] = self.fallback_to_all(suffix)
elif name.endswith(".rpm"):
suffix = self.rpm.pop(name)
self.rpm[self.fallback_to_all(name)] = self.fallback_to_all(suffix)
elif name.endswith(".tgz"):
suffix = self.tgz.pop(name)
self.tgz[self.fallback_to_all(name)] = self.fallback_to_all(suffix)
else:
raise KeyError(f"unknown package type for {name}")
@staticmethod
def path(package_file: str) -> str:
return os.path.join(TEMP_PATH, package_file)
@staticmethod
def fallback_to_all(url_or_name: str):
"""Until July 2022 we had clickhouse-server and clickhouse-client with
arch 'all'"""
# deb
if url_or_name.endswith("amd64.deb") or url_or_name.endswith("arm64.deb"):
return f"{url_or_name[:-9]}all.deb"
# rpm
if url_or_name.endswith("x86_64.rpm") or url_or_name.endswith("aarch64.rpm"):
new = removesuffix(removesuffix(url_or_name, "x86_64.rpm"), "aarch64.rpm")
return f"{new}noarch.rpm"
# tgz
if url_or_name.endswith("-amd64.tgz") or url_or_name.endswith("-arm64.tgz"):
return f"{url_or_name[:-10]}.tgz"
return url_or_name
class S3:
template = (
"https://s3.amazonaws.com/"
# "clickhouse-builds/"
"{bucket_name}/"
f"{S3_BUILDS_BUCKET}/"
# "33333/" or "21.11/" from --release, if pull request is omitted
"{pr}/"
# "2bef313f75e4cacc6ea2ef2133e8849ecf0385ec/"
"{commit}/"
# "package_release/"
"{check_name}/"
# "clickhouse-common-static_21.11.5.0_amd64.deb"
"{package}"
# "package_release/clickhouse-common-static_21.11.5.0_amd64.deb"
"{s3_path_suffix}"
)
def __init__(
self,
bucket_name: str,
pr: int,
commit: str,
check_name: str,
version: str,
force_download: bool,
):
self._common = dict(
bucket_name=bucket_name,
pr=pr,
commit=commit,
check_name=check_name,
)
self.force_download = force_download
self.packages = Packages(version)
def download_package(self, package_file: str):
if not self.force_download and os.path.exists(Packages.path(package_file)):
def download_package(self, package_file: str, s3_path_suffix: str):
path = Packages.path(package_file)
fallback_path = Packages.fallback_to_all(path)
if not self.force_download and (
os.path.exists(path) or os.path.exists(fallback_path)
):
if os.path.exists(fallback_path):
self.packages.replace_with_fallback(package_file)
return
url = self.template.format_map({**self._common, "package": package_file})
dowload_build_with_progress(url, Packages.path(package_file))
url = self.template.format_map(
{**self._common, "s3_path_suffix": s3_path_suffix}
)
try:
dowload_build_with_progress(url, path)
except Exception as e:
if "Cannot download dataset from" in e.args[0]:
new_url = Packages.fallback_to_all(url)
logging.warning(
"Fallback downloading %s for old release", fallback_path
)
dowload_build_with_progress(new_url, fallback_path)
self.packages.replace_with_fallback(package_file)
def download_deb(self):
for package_file in self.packages.deb:
self.download_package(package_file)
# Copy to have a way to pop/add fallback packages
packages = self.packages.deb.copy()
for package_file, s3_path_suffix in packages.items():
self.download_package(package_file, s3_path_suffix)
def download_rpm(self):
for package_file in self.packages.rpm:
self.download_package(package_file)
# Copy to have a way to pop/add fallback packages
packages = self.packages.rpm.copy()
for package_file, s3_path_suffix in packages.items():
self.download_package(package_file, s3_path_suffix)
def download_tgz(self):
for package_file in self.packages.tgz:
self.download_package(package_file)
# Copy to have a way to pop/add fallback packages
packages = self.packages.tgz.copy()
for package_file, s3_path_suffix in packages.items():
self.download_package(package_file, s3_path_suffix)
class Release:
@ -223,17 +280,6 @@ def parse_args() -> argparse.Namespace:
parser.add_argument(
"--commit", required=True, type=commit, help="commit hash for S3 bucket"
)
parser.add_argument(
"--bucket-name",
default="clickhouse-builds",
help="AWS S3 bucket name",
)
parser.add_argument(
"--check-name",
default="package_release",
help="check name, a part of bucket path, "
"will be converted to lower case with spaces->underscore",
)
parser.add_argument(
"--all", action="store_true", help="implies all deb, rpm and tgz"
)
@ -276,7 +322,6 @@ def parse_args() -> argparse.Namespace:
args.deb = args.rpm = args.tgz = True
if not (args.deb or args.rpm or args.tgz):
parser.error("at least one of --deb, --rpm or --tgz should be specified")
args.check_name = args.check_name.lower().replace(" ", "_")
if args.pull_request == 0:
args.pull_request = ".".join(args.release.version_parts[:2])
return args
@ -305,10 +350,8 @@ def main():
args = parse_args()
os.makedirs(TEMP_PATH, exist_ok=True)
s3 = S3(
args.bucket_name,
args.pull_request,
args.commit,
args.check_name,
args.release.version,
args.force_download,
)