Use a proper way to download packages from releases

docker/test/upgrade/run.sh

@@ -25,7 +25,8 @@ azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
 ./setup_minio.sh stateless # to have a proper environment
 
 echo "Get previous release tag"
-previous_release_tag=$(dpkg --info package_folder/clickhouse-client*.deb | grep "Version: " | awk '{print $2}' | cut -f1 -d'+' | get_previous_release_tag)
+# shellcheck disable=SC2016
+previous_release_tag=$(dpkg-deb --showformat='${Version}' --show package_folder/clickhouse-client*.deb | get_previous_release_tag)
 echo $previous_release_tag
 
 echo "Clone previous release repository"

tests/ci/build_download_helper.py

@@ -10,9 +10,21 @@ from typing import Any, Callable, List, Optional, Union
 
 import requests
 
-import get_robot_token as grt  # we need an updated ROBOT_TOKEN
 from ci_config import CI_CONFIG
 
+try:
+    # A work around for scripts using this downloading module without required deps
+    import get_robot_token as grt  # we need an updated ROBOT_TOKEN
+except ImportError:
+
+    class grt:  # type: ignore
+        ROBOT_TOKEN = None
+
+        @staticmethod
+        def get_best_robot_token() -> str:
+            return ""
+
+
 DOWNLOAD_RETRIES_COUNT = 5
 
 

tests/ci/download_release_packages.py

@@ -1,79 +1,38 @@
 #!/usr/bin/env python3
 
 import logging
-import os
+from pathlib import Path
 
-import requests
-from requests.adapters import HTTPAdapter  # type: ignore
-from urllib3.util.retry import Retry  # type: ignore
-
-from get_previous_release_tag import ReleaseInfo, get_previous_release
-
-CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags"
-
-DOWNLOAD_PREFIX = (
-    "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/"
+from build_download_helper import DownloadException, download_build_with_progress
+from get_previous_release_tag import (
+    ReleaseInfo,
+    get_previous_release,
+    get_release_by_tag,
 )
-CLICKHOUSE_COMMON_STATIC_PACKAGE_NAME = "clickhouse-common-static_{version}_amd64.deb"
-CLICKHOUSE_COMMON_STATIC_DBG_PACKAGE_NAME = (
-    "clickhouse-common-static-dbg_{version}_amd64.deb"
-)
-CLICKHOUSE_CLIENT_PACKAGE_NAME = "clickhouse-client_{version}_amd64.deb"
-CLICKHOUSE_LIBRARY_BRIDGE_PACKAGE_NAME = "clickhouse-library-bridge_{version}_amd64.deb"
-CLICKHOUSE_ODBC_BRIDGE_PACKAGE_NAME = "clickhouse-odbc-bridge_{version}_amd64.deb"
-CLICKHOUSE_SERVER_PACKAGE_NAME = "clickhouse-server_{version}_amd64.deb"
 
-PACKAGES_DIR = "previous_release_package_folder/"
-VERSION_PATTERN = r"((?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)"
+PACKAGES_DIR = Path("previous_release_package_folder")
 
 
-def download_package(url, out_path, retries=10, backoff_factor=0.3):
-    session = requests.Session()
-    retry = Retry(
-        total=retries,
-        read=retries,
-        connect=retries,
-        backoff_factor=backoff_factor,
-        status_forcelist=[500, 502, 503, 504],
-    )
-    adapter = HTTPAdapter(max_retries=retry)
-    session.mount("http://", adapter)
-    session.mount("https://", adapter)
-    response = session.get(url)
-    response.raise_for_status()
-    print(f"Download {url} to {out_path}")
-    with open(out_path, "wb") as fd:
-        fd.write(response.content)
-
-
-def download_packages(release, dest_path=PACKAGES_DIR):
-    if not os.path.exists(dest_path):
-        os.makedirs(dest_path)
+def download_packages(release: ReleaseInfo, dest_path: Path = PACKAGES_DIR) -> None:
+    dest_path.mkdir(parents=True, exist_ok=True)
 
     logging.info("Will download %s", release)
 
-    def get_dest_path(pkg_name):
-        return os.path.join(dest_path, pkg_name)
-
-    for pkg in (
-        CLICKHOUSE_COMMON_STATIC_PACKAGE_NAME,
-        CLICKHOUSE_COMMON_STATIC_DBG_PACKAGE_NAME,
-        CLICKHOUSE_CLIENT_PACKAGE_NAME,
-        CLICKHOUSE_LIBRARY_BRIDGE_PACKAGE_NAME,
-        CLICKHOUSE_ODBC_BRIDGE_PACKAGE_NAME,
-        CLICKHOUSE_SERVER_PACKAGE_NAME,
-    ):
-        url = (DOWNLOAD_PREFIX + pkg).format(version=release.version, type=release.type)
-        pkg_name = get_dest_path(pkg.format(version=release.version))
-        download_package(url, pkg_name)
+    for pkg, url in release.assets.items():
+        if not pkg.endswith("_amd64.deb") or "-dbg_" in pkg:
+            continue
+        pkg_name = dest_path / pkg
+        download_build_with_progress(url, pkg_name)
 
 
-def download_last_release(dest_path):
+def download_last_release(dest_path: Path) -> None:
     current_release = get_previous_release(None)
+    if current_release is None:
+        raise DownloadException("The current release is not found")
     download_packages(current_release, dest_path=dest_path)
 
 
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
-    release = ReleaseInfo(input())
+    release = get_release_by_tag(input())
     download_packages(release)

tests/ci/get_previous_release_tag.py

@@ -2,47 +2,37 @@
 
 import logging
 import re
-from typing import List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple
 
-import requests
-
-CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags"
-CLICKHOUSE_PACKAGE_URL = (
-    "https://github.com/ClickHouse/ClickHouse/releases/download/"
-    "v{version}-{type}/clickhouse-common-static_{version}_amd64.deb"
+from build_download_helper import get_gh_api
+from git_helper import TAG_REGEXP
+from version_helper import (
+    ClickHouseVersion,
+    get_version_from_string,
+    get_version_from_tag,
 )
-VERSION_PATTERN = r"(v(?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)"
+
+CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/releases"
+PACKAGE_REGEXP = r"\Aclickhouse-common-static_.+[.]deb"
 
 logger = logging.getLogger(__name__)
 
 
-class Version:
-    def __init__(self, version: str):
-        self.version = version
-
-    def __lt__(self, other: "Version") -> bool:
-        return list(map(int, self.version.split("."))) < list(
-            map(int, other.version.split("."))
-        )
-
-    def __str__(self):
-        return self.version
-
-
 class ReleaseInfo:
-    def __init__(self, release_tag: str):
-        self.version = Version(release_tag[1:].split("-")[0])
-        self.type = release_tag[1:].split("-")[1]
+    def __init__(self, release_tag: str, assets: Dict[str, str]):
+        self.version = get_version_from_tag(release_tag)
+        self.type = self.version.description
+        self.assets = assets
 
     def __str__(self):
-        return f"v{self.version}-{self.type}"
+        return self.version.describe
 
     def __repr__(self):
-        return f"ReleaseInfo: {self.version}-{self.type}"
+        return f"ReleaseInfo: {self.version.describe}"
 
 
 def find_previous_release(
-    server_version: Optional[Version], releases: List[ReleaseInfo]
+    server_version: Optional[ClickHouseVersion], releases: List[ReleaseInfo]
 ) -> Tuple[bool, Optional[ReleaseInfo]]:
     releases.sort(key=lambda x: x.version, reverse=True)
 
@@ -54,15 +44,7 @@ def find_previous_release(
             # Check if the artifact exists on GitHub.
             # It can be not true for a short period of time
             # after creating a tag for a new release before uploading the packages.
-            if (
-                requests.head(
-                    CLICKHOUSE_PACKAGE_URL.format(
-                        version=release.version, type=release.type
-                    ),
-                    timeout=10,
-                ).status_code
-                != 404
-            ):
+            if any(re.match(PACKAGE_REGEXP, name) for name in release.assets.keys()):
                 return True, release
 
             logger.debug(
@@ -74,12 +56,14 @@ def find_previous_release(
     return False, None
 
 
-def get_previous_release(server_version: Optional[Version]) -> Optional[ReleaseInfo]:
+def get_previous_release(
+    server_version: Optional[ClickHouseVersion],
+) -> Optional[ReleaseInfo]:
     page = 1
     found = False
     while not found:
-        response = requests.get(
-            CLICKHOUSE_TAGS_URL, {"page": page, "per_page": 100}, timeout=10
+        response = get_gh_api(
+            CLICKHOUSE_TAGS_URL, params={"page": page, "per_page": 100}, timeout=10
         )
         if not response.ok:
             logger.error(
@@ -87,24 +71,42 @@ def get_previous_release(server_version: Optional[Version]) -> Optional[ReleaseI
             )
             response.raise_for_status()
 
-        releases_str = set(re.findall(VERSION_PATTERN, response.text))
-        if len(releases_str) == 0:
-            raise ValueError(
-                "Cannot find previous release for "
-                + str(server_version)
-                + " server version"
-            )
+        releases = response.json()
 
-        releases = [ReleaseInfo(release) for release in releases_str]
-        found, previous_release = find_previous_release(server_version, releases)
+        release_infos = []  # type: List[ReleaseInfo]
+        for r in releases:
+            if re.match(TAG_REGEXP, r["tag_name"]):
+                assets = {
+                    a["name"]: a["browser_download_url"]
+                    for a in r["assets"]
+                    if a["state"] == "uploaded"
+                }
+                release_infos.append(ReleaseInfo(r["tag_name"], assets))
+        found, previous_release = find_previous_release(server_version, release_infos)
         page += 1
 
     return previous_release
 
 
+def get_release_by_tag(tag: str) -> ReleaseInfo:
+    response = get_gh_api(f"{CLICKHOUSE_TAGS_URL}/tags/{tag}", timeout=10)
+    release = response.json()
+    assets = {
+        a["name"]: a["browser_download_url"]
+        for a in release["assets"]
+        if a["state"] == "uploaded"
+    }
+    return ReleaseInfo(release["tag_name"], assets)
+
+
 def main():
     logging.basicConfig(level=logging.INFO)
-    server_version = Version(input())
+    version_string = input()
+    version_string = version_string.split("+", maxsplit=1)[0]
+    try:
+        server_version = get_version_from_string(version_string)
+    except ValueError:
+        server_version = get_version_from_tag(version_string)
     print(get_previous_release(server_version))