#!/usr/bin/env python3 # -*- coding: utf-8 -*- import argparse import logging import os import re from collections import namedtuple from typing import Dict, List, Tuple from artifactory import ArtifactorySaaSPath # type: ignore from build_download_helper import dowload_build_with_progress from env_helper import RUNNER_TEMP, S3_BUILDS_BUCKET, S3_DOWNLOAD from git_helper import TAG_REGEXP, commit, removeprefix, removesuffix # Necessary ENV variables def getenv(name: str, default: str = None): env = os.getenv(name, default) if env is not None: return env raise KeyError(f"Necessary {name} environment is not set") TEMP_PATH = os.path.join(RUNNER_TEMP, "push_to_artifactory") # One of the following ENVs is necessary JFROG_API_KEY = getenv("JFROG_API_KEY", "") JFROG_TOKEN = getenv("JFROG_TOKEN", "") CheckDesc = namedtuple("CheckDesc", ("check_name", "deb_arch", "rpm_arch")) class Packages: checks = ( CheckDesc("package_release", "amd64", "x86_64"), CheckDesc("package_aarch64", "arm64", "aarch64"), ) packages = ( "clickhouse-client", "clickhouse-common-static", "clickhouse-common-static-dbg", "clickhouse-server", ) def __init__(self, version: str): # Dicts of name: s3_path_suffix self.deb = {} # type: Dict[str, str] self.rpm = {} # type: Dict[str, str] self.tgz = {} # type: Dict[str, str] for check in self.checks: for name in self.packages: deb = f"{name}_{version}_{check.deb_arch}.deb" self.deb[deb] = f"{check.check_name}/{deb}" rpm = f"{name}-{version}.{check.rpm_arch}.rpm" self.rpm[rpm] = f"{check.check_name}/{rpm}" tgz = f"{name}-{version}-{check.deb_arch}.tgz" self.tgz[tgz] = f"{check.check_name}/{tgz}" def arch(self, deb_pkg: str) -> str: if deb_pkg not in self.deb: raise ValueError(f"{deb_pkg} not in {self.deb}") return removesuffix(deb_pkg, ".deb").split("_")[-1] def replace_with_fallback(self, name: str): if name.endswith(".deb"): suffix = self.deb.pop(name) self.deb[self.fallback_to_all(name)] = self.fallback_to_all(suffix) elif name.endswith(".rpm"): suffix = self.rpm.pop(name) self.rpm[self.fallback_to_all(name)] = self.fallback_to_all(suffix) elif name.endswith(".tgz"): suffix = self.tgz.pop(name) self.tgz[self.fallback_to_all(name)] = self.fallback_to_all(suffix) else: raise KeyError(f"unknown package type for {name}") @staticmethod def path(package_file: str) -> str: return os.path.join(TEMP_PATH, package_file) @staticmethod def fallback_to_all(url_or_name: str): """Until July 2022 we had clickhouse-server and clickhouse-client with arch 'all'""" # deb if url_or_name.endswith("amd64.deb") or url_or_name.endswith("arm64.deb"): return f"{url_or_name[:-9]}all.deb" # rpm if url_or_name.endswith("x86_64.rpm") or url_or_name.endswith("aarch64.rpm"): new = removesuffix(removesuffix(url_or_name, "x86_64.rpm"), "aarch64.rpm") return f"{new}noarch.rpm" # tgz if url_or_name.endswith("-amd64.tgz") or url_or_name.endswith("-arm64.tgz"): return f"{url_or_name[:-10]}.tgz" return url_or_name class S3: template = ( f"{S3_DOWNLOAD}" # "clickhouse-builds/" f"{S3_BUILDS_BUCKET}/" # "33333/" or "21.11/" from --release, if pull request is omitted "{pr}/" # "2bef313f75e4cacc6ea2ef2133e8849ecf0385ec/" "{commit}/" # "package_release/clickhouse-common-static_21.11.5.0_amd64.deb" "{s3_path_suffix}" ) def __init__( self, pr: int, commit: str, version: str, force_download: bool, ): self._common = dict( pr=pr, commit=commit, ) self.force_download = force_download self.packages = Packages(version) def download_package(self, package_file: str, s3_path_suffix: str): path = Packages.path(package_file) fallback_path = Packages.fallback_to_all(path) if not self.force_download and ( os.path.exists(path) or os.path.exists(fallback_path) ): if os.path.exists(fallback_path): self.packages.replace_with_fallback(package_file) return url = self.template.format_map( {**self._common, "s3_path_suffix": s3_path_suffix} ) try: dowload_build_with_progress(url, path) except Exception as e: if "Cannot download dataset from" in e.args[0]: new_url = Packages.fallback_to_all(url) logging.warning( "Fallback downloading %s for old release", fallback_path ) dowload_build_with_progress(new_url, fallback_path) self.packages.replace_with_fallback(package_file) def download_deb(self): # Copy to have a way to pop/add fallback packages packages = self.packages.deb.copy() for package_file, s3_path_suffix in packages.items(): self.download_package(package_file, s3_path_suffix) def download_rpm(self): # Copy to have a way to pop/add fallback packages packages = self.packages.rpm.copy() for package_file, s3_path_suffix in packages.items(): self.download_package(package_file, s3_path_suffix) def download_tgz(self): # Copy to have a way to pop/add fallback packages packages = self.packages.tgz.copy() for package_file, s3_path_suffix in packages.items(): self.download_package(package_file, s3_path_suffix) class Release: def __init__(self, name: str): r = re.compile(TAG_REGEXP) # Automatically remove refs/tags/ if full refname passed here name = removeprefix(name, "refs/tags/") if not r.match(name): raise argparse.ArgumentTypeError( f"release name {name} does not match " "v12.1.2.15-(testing|prestable|stable|lts) pattern" ) self._name = name self._version = removeprefix(self._name, "v") self._version = self.version.split("-")[0] self._version_parts = tuple(self.version.split(".")) self._type = self._name.split("-")[-1] @property def version(self) -> str: return self._version @property def version_parts(self) -> Tuple[str, ...]: return self._version_parts @property def type(self) -> str: return self._type class Artifactory: def __init__( self, url: str, release: str, deb_repo="deb", rpm_repo="rpm", tgz_repo="tgz" ): self._url = url self._release = release self._deb_url = "/".join((self._url, deb_repo, "pool", self._release)) + "/" self._rpm_url = "/".join((self._url, rpm_repo, self._release)) + "/" self._tgz_url = "/".join((self._url, tgz_repo, self._release)) + "/" # check the credentials ENVs for early exit self.__path_helper("_deb", "") def deploy_deb(self, packages: Packages): for package_file in packages.deb: path = packages.path(package_file) dist = self._release comp = "main" arch = packages.arch(package_file) logging.info( "Deploy %s(distribution=%s;component=%s;architecture=%s) " "to artifactory", path, dist, comp, arch, ) self.deb_path(package_file).deploy_deb(path, dist, comp, arch) def deploy_rpm(self, packages: Packages): for package_file in packages.rpm: path = packages.path(package_file) logging.info("Deploy %s to artifactory", path) self.rpm_path(package_file).deploy_file(path) def deploy_tgz(self, packages: Packages): for package_file in packages.tgz: path = packages.path(package_file) logging.info("Deploy %s to artifactory", path) self.tgz_path(package_file).deploy_file(path) def __path_helper(self, name: str, package_file: str) -> ArtifactorySaaSPath: url = "/".join((getattr(self, name + "_url"), package_file)) path = None if JFROG_API_KEY: path = ArtifactorySaaSPath(url, apikey=JFROG_API_KEY) elif JFROG_TOKEN: path = ArtifactorySaaSPath(url, token=JFROG_TOKEN) else: raise KeyError("Neither JFROG_API_KEY nor JFROG_TOKEN env are defined") return path def deb_path(self, package_file: str) -> ArtifactorySaaSPath: return self.__path_helper("_deb", package_file) def rpm_path(self, package_file: str) -> ArtifactorySaaSPath: return self.__path_helper("_rpm", package_file) def tgz_path(self, package_file: str) -> ArtifactorySaaSPath: return self.__path_helper("_tgz", package_file) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description="Program to download artifacts from S3 and push them to " "artifactory. ENV variables JFROG_API_KEY and JFROG_TOKEN are used " "for authentication in the given order", ) parser.add_argument( "--release", required=True, type=Release, help="release name, e.g. v12.13.14.15-prestable; 'refs/tags/' " "prefix is striped automatically", ) parser.add_argument( "--pull-request", type=int, default=0, help="pull request number; if PR is omitted, the first two numbers " "from release will be used, e.g. 12.11", ) parser.add_argument( "--commit", required=True, type=commit, help="commit hash for S3 bucket" ) parser.add_argument( "--all", action="store_true", help="implies all deb, rpm and tgz" ) parser.add_argument( "--deb", action="store_true", help="if Debian packages should be processed" ) parser.add_argument( "--rpm", action="store_true", help="if RPM packages should be processed" ) parser.add_argument( "--tgz", action="store_true", help="if tgz archives should be processed. They aren't pushed to artifactory", ) parser.add_argument( "--artifactory-url", default="https://clickhousedb.jfrog.io/artifactory", help="SaaS Artifactory url", ) parser.add_argument("--artifactory", default=True, help=argparse.SUPPRESS) parser.add_argument( "-n", "--no-artifactory", action="store_false", dest="artifactory", default=argparse.SUPPRESS, help="do not push packages to artifactory", ) parser.add_argument("--force-download", default=True, help=argparse.SUPPRESS) parser.add_argument( "--no-force-download", action="store_false", dest="force_download", default=argparse.SUPPRESS, help="do not download packages again if they exist already", ) args = parser.parse_args() if args.all: args.deb = args.rpm = args.tgz = True if not (args.deb or args.rpm or args.tgz): parser.error("at least one of --deb, --rpm or --tgz should be specified") if args.pull_request == 0: args.pull_request = ".".join(args.release.version_parts[:2]) return args def process_deb(s3: S3, art_clients: List[Artifactory]): s3.download_deb() for art_client in art_clients: art_client.deploy_deb(s3.packages) def process_rpm(s3: S3, art_clients: List[Artifactory]): s3.download_rpm() for art_client in art_clients: art_client.deploy_rpm(s3.packages) def process_tgz(s3: S3, art_clients: List[Artifactory]): s3.download_tgz() for art_client in art_clients: art_client.deploy_tgz(s3.packages) def main(): logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") args = parse_args() os.makedirs(TEMP_PATH, exist_ok=True) s3 = S3( args.pull_request, args.commit, args.release.version, args.force_download, ) art_clients = [] if args.artifactory: art_clients.append(Artifactory(args.artifactory_url, args.release.type)) if args.release.type == "lts": art_clients.append(Artifactory(args.artifactory_url, "stable")) if args.deb: process_deb(s3, art_clients) if args.rpm: process_rpm(s3, art_clients) if args.tgz: process_tgz(s3, art_clients) if __name__ == "__main__": main()