Change clickhouse-{client,server} packages architecture

2024-11-21 15:12:02 +00:00 · 2022-06-29 12:33:20 +02:00 · 2022-06-29 12:33:20 +02:00 · e76ebdfb47
commit e76ebdfb47
parent 80e7abd0fe
9 changed files with 131 additions and 92 deletions
--- a/docs/_includes/install/tgz.sh
+++ b/docs/_includes/install/tgz.sh
@ -1,20 +1,34 @@
 LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
    grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
 export LATEST_VERSION
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz"

-tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
+case $(uname -m) in
+  x86_64) ARCH=amd64 ;;
+  aarch64) ARCH=arm64 ;;
+  *) echo "Unknown architecture $(uname -m)"; exit 1 ;;
+esac
+
+for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client
+do
+  curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \
+    || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz"
+done
+
+exit 0
+
+tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh"

-tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh"

-tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
 sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh"
 sudo /etc/init.d/clickhouse-server start

-tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
 sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh"
--- a/docs/en/development/continuous-integration.md
+++ b/docs/en/development/continuous-integration.md
@ -123,12 +123,10 @@ Builds ClickHouse in various configurations for use in further steps. You have t
 - **Build log**: link to the building and files copying log, useful when build failed.
 - **Build time**.
 - **Artifacts**: build result files (with `XXX` being the server version e.g. `20.8.1.4344`).
-  - `clickhouse-client_XXX_all.deb`
+  - `clickhouse-client_XXX_amd64.deb`
  - `clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb`
  - `clickhouse-common-staticXXX_amd64.deb`
-  - `clickhouse-server_XXX_all.deb`
-  - `clickhouse_XXX_amd64.buildinfo`
-  - `clickhouse_XXX_amd64.changes`
+  - `clickhouse-server_XXX_amd64.deb`
  - `clickhouse`: Main built binary.
  - `clickhouse-odbc-bridge`
  - `unit_tests_dbms`: GoogleTest binary with ClickHouse unit tests.
--- a/docs/en/getting-started/install.md
+++ b/docs/en/getting-started/install.md
@ -127,22 +127,36 @@ After that downloaded archives should be unpacked and installed with installatio
 LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
    grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
 export LATEST_VERSION
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION-amd64.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION-amd64.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION-amd64.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION-amd64.tgz"

-tar -xzvf "clickhouse-common-static-$LATEST_VERSION-amd64.tgz"
+case $(uname -m) in
+  x86_64) ARCH=amd64 ;;
+  aarch64) ARCH=arm64 ;;
+  *) echo "Unknown architecture $(uname -m)"; exit 1 ;;
+esac
+
+for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client
+do
+  curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \
+    || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz"
+done
+
+exit 0
+
+tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh"

-tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-amd64.tgz"
+tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh"

-tar -xzvf "clickhouse-server-$LATEST_VERSION-amd64.tgz"
+tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
 sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh"
 sudo /etc/init.d/clickhouse-server start

-tar -xzvf "clickhouse-client-$LATEST_VERSION-amd64.tgz"
+tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
 sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh"
 ```

--- a/docs/ru/getting-started/install.md
+++ b/docs/ru/getting-started/install.md
@ -124,22 +124,36 @@ sudo yum install clickhouse-server clickhouse-client
 LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
    grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
 export LATEST_VERSION
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz"

-tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
+case $(uname -m) in
+  x86_64) ARCH=amd64 ;;
+  aarch64) ARCH=arm64 ;;
+  *) echo "Unknown architecture $(uname -m)"; exit 1 ;;
+esac
+
+for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client
+do
+  curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \
+    || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz"
+done
+
+exit 0
+
+tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh"

-tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh"

-tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
 sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh"
 sudo /etc/init.d/clickhouse-server start

-tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
 sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh"
 ```

--- a/docs/zh/development/continuous-integration.md
+++ b/docs/zh/development/continuous-integration.md
@ -86,13 +86,10 @@ git push
 - **Build log**: 链接到构建和文件复制日志, 当构建失败时很有用.
 - **Build time**.
 - **Artifacts**: 构建结果文件 (`XXX`是服务器版本, 比如`20.8.1.4344`).
-    - `clickhouse-client_XXX_all.deb`
+    - `clickhouse-client_XXX_amd64.deb`
    -` clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb`
    - `clickhouse-common-staticXXX_amd64.deb`
-    - `clickhouse-server_XXX_all.deb`
-    - `clickhouse-test_XXX_all.deb`
-    - `clickhouse_XXX_amd64.buildinfo`
-    - `clickhouse_XXX_amd64.changes`
+    - `clickhouse-server_XXX_amd64.deb`
    - `clickhouse`: Main built binary.
    - `clickhouse-odbc-bridge`
    - `unit_tests_dbms`: 带有 ClickHouse 单元测试的 GoogleTest 二进制文件.
--- a/docs/zh/getting-started/install.md
+++ b/docs/zh/getting-started/install.md
@ -121,22 +121,36 @@ sudo yum install clickhouse-server clickhouse-client
 LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
    grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
 export LATEST_VERSION
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz"

-tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
+case $(uname -m) in
+  x86_64) ARCH=amd64 ;;
+  aarch64) ARCH=arm64 ;;
+  *) echo "Unknown architecture $(uname -m)"; exit 1 ;;
+esac
+
+for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client
+do
+  curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \
+    || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz"
+done
+
+exit 0
+
+tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh"

-tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh"

-tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
 sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh"
 sudo /etc/init.d/clickhouse-server start

-tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
 sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh"
 ```

--- a/packages/clickhouse-client.yaml
+++ b/packages/clickhouse-client.yaml
@ -1,7 +1,7 @@
 # package sources should be placed in ${PWD}/root
 # nfpm should run from the same directory with a config
 name: "clickhouse-client"
-arch: "all"
+arch: "${DEB_ARCH}" # amd64, arm64
 platform: "linux"
 version: "${CLICKHOUSE_VERSION_STRING}"
 vendor: "ClickHouse Inc."
--- a/packages/clickhouse-server.yaml
+++ b/packages/clickhouse-server.yaml
@ -1,7 +1,7 @@
 # package sources should be placed in ${PWD}/root
 # nfpm should run from the same directory with a config
 name: "clickhouse-server"
-arch: "all"
+arch: "${DEB_ARCH}" # amd64, arm64
 platform: "linux"
 version: "${CLICKHOUSE_VERSION_STRING}"
 vendor: "ClickHouse Inc."
--- a/tests/ci/download_previous_release.py
+++ b/tests/ci/download_previous_release.py
@ -4,25 +4,24 @@ import re
 import os
 import logging

-import requests
+import requests  # type: ignore

-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
+from requests.adapters import HTTPAdapter  # type: ignore
+from urllib3.util.retry import Retry  # type: ignore

 CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags"

-CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb"
-CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static-dbg_{version}_amd64.deb"
-CLICKHOUSE_SERVER_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-server_{version}_all.deb"
-CLICKHOUSE_CLIENT_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-client_{version}_all.deb"
-
-
+DOWNLOAD_PREFIX = (
+    "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/"
+)
 CLICKHOUSE_COMMON_STATIC_PACKET_NAME = "clickhouse-common-static_{version}_amd64.deb"
 CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME = (
    "clickhouse-common-static-dbg_{version}_amd64.deb"
 )
-CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_all.deb"
-CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_all.deb"
+CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_amd64.deb"
+CLICKHOUSE_SERVER_PACKET_FALLBACK = "clickhouse-server_{version}_all.deb"
+CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_amd64.deb"
+CLICKHOUSE_CLIENT_PACKET_FALLBACK = "clickhouse-client_{version}_all.deb"

 PACKETS_DIR = "previous_release_package_folder/"
 VERSION_PATTERN = r"((?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)"
@ -102,9 +101,10 @@ def download_packet(url, out_path, retries=10, backoff_factor=0.3):
    session.mount("http://", adapter)
    session.mount("https://", adapter)
    response = session.get(url)
-    print(url)
-    if response.ok:
-        open(out_path, "wb").write(response.content)
+    response.raise_for_status()
+    print(f"Download {url} to {out_path}")
+    with open(out_path, "wb") as fd:
+        fd.write(response.content)


 def download_packets(release, dest_path=PACKETS_DIR):
@ -113,43 +113,31 @@ def download_packets(release, dest_path=PACKETS_DIR):

    logging.info("Will download %s", release)

-    download_packet(
-        CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format(
-            version=release.version, type=release.type
-        ),
-        out_path=os.path.join(
-            dest_path,
-            CLICKHOUSE_COMMON_STATIC_PACKET_NAME.format(version=release.version),
-        ),
-    )
+    def get_dest_path(pkg_name):
+        return os.path.join(dest_path, pkg_name)

-    download_packet(
-        CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL.format(
-            version=release.version, type=release.type
-        ),
-        out_path=os.path.join(
-            dest_path,
-            CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME.format(version=release.version),
-        ),
-    )
+    for pkg in (
+        CLICKHOUSE_COMMON_STATIC_PACKET_NAME,
+        CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME,
+    ):
+        url = (DOWNLOAD_PREFIX + pkg).format(version=release.version, type=release.type)
+        pkg_name = get_dest_path(pkg.format(version=release.version))
+        download_packet(url, pkg_name)

-    download_packet(
-        CLICKHOUSE_SERVER_DOWNLOAD_URL.format(
-            version=release.version, type=release.type
-        ),
-        out_path=os.path.join(
-            dest_path, CLICKHOUSE_SERVER_PACKET_NAME.format(version=release.version)
-        ),
-    )
-
-    download_packet(
-        CLICKHOUSE_CLIENT_DOWNLOAD_URL.format(
-            version=release.version, type=release.type
-        ),
-        out_path=os.path.join(
-            dest_path, CLICKHOUSE_CLIENT_PACKET_NAME.format(version=release.version)
-        ),
-    )
+    for pkg, fallback in (
+        (CLICKHOUSE_SERVER_PACKET_NAME, CLICKHOUSE_SERVER_PACKET_FALLBACK),
+        (CLICKHOUSE_CLIENT_PACKET_NAME, CLICKHOUSE_CLIENT_PACKET_FALLBACK),
+    ):
+        url = (DOWNLOAD_PREFIX + pkg).format(version=release.version, type=release.type)
+        pkg_name = get_dest_path(pkg.format(version=release.version))
+        try:
+            download_packet(url, pkg_name)
+        except Exception:
+            url = (DOWNLOAD_PREFIX + fallback).format(
+                version=release.version, type=release.type
+            )
+            pkg_name = get_dest_path(fallback.format(version=release.version))
+            download_packet(url, pkg_name)


 def download_previous_release(dest_path):