#!/usr/bin/env python3 # -*- coding: utf-8 -*- import argparse import logging import os import re from artifactory import ArtifactorySaaSPath from build_download_helper import dowload_build_with_progress # Necessary ENV variables def getenv(name, default=None): env = os.getenv(name, default) if env is not None: return env raise KeyError(f"Necessary {name} environment is not set") TEMP_PATH = getenv("TEMP_PATH", ".") # One of the following ENVs is necessary JFROG_API_KEY = getenv("JFROG_API_KEY", "") JFROG_TOKEN = getenv("JFROG_TOKEN", "") class Packages(object): rpm_arch = dict(all="noarch", amd64="x86_64") packages = ( ("clickhouse-client", "all"), ("clickhouse-common-static", "amd64"), ("clickhouse-common-static-dbg", "amd64"), ("clickhouse-server", "all"), ("clickhouse-test", "all"), ) def __init__(self, version: str): self.deb = tuple( "_".join((name, version, arch + ".deb")) for name, arch in self.packages ) rev = "2" self.rpm = tuple( "-".join((name, version, rev + "." + self.rpm_arch[arch] + ".rpm")) for name, arch in self.packages ) def arch(self, deb_pkg: str) -> str: if deb_pkg not in self.deb: raise ValueError("{} not in {}".format(deb_pkg, self.deb)) return deb_pkg.removesuffix(".deb").split("_")[-1] @staticmethod def path(package): return os.path.join(TEMP_PATH, package) class S3(object): template = ( "https://s3.amazonaws.com/" # "clickhouse-builds/" "{bucket_name}/" # "33333/" or "0/" "{pr}/" # "2bef313f75e4cacc6ea2ef2133e8849ecf0385ec/" "{commit}/" # "clickhouse_build_check_(actions)/" "{check_name}/" # "clang-13_relwithdebuginfo_memory_bundled_unsplitted_notidy_without_coverage_deb/" "{build_name}/" # "clickhouse-common-static_21.11.5.0_amd64.deb" "{package}" ) def __init__( self, bucket_name: str, pr: int, commit: str, check_name: str, build_name: str, version: str, ): self._common = dict( bucket_name=bucket_name, pr=pr, commit=commit, check_name=check_name, build_name=build_name, ) self.packages = Packages(version) def download_package(self, package): url = self.template.format_map({**self._common, "package": package}) dowload_build_with_progress(url, Packages.path(package)) def download_deb(self): for package in self.packages.deb: self.download_package(package) def download_rpm(self): for package in self.packages.rpm: self.download_package(package) class Release(object): def __call__(self, name: str) -> str: r = re.compile(r"^v\d{2}[.]\d+[.]\d+[.]\d+-(testing|prestable|stable|lts)$") if not r.match(name): raise argparse.ArgumentTypeError( "release name does not match v12.13.14-TYPE pattern" ) self._name = name return self @property def version(self) -> str: if getattr(self, "_version", False): return self._version version = self._name.removeprefix("v") self._version = version.split("-")[0] return self._version @property def type(self) -> str: if getattr(self, "_type", False): return self._type self._type = self._name.split("-")[-1] return self._type class Artifactory(object): def __init__(self, url: str, release: str, deb_repo="deb", rpm_repo="rpm"): self._url = url self._release = release self._deb_url = "/".join((self._url, deb_repo, "pool", self._release)) + "/" self._rpm_url = "/".join((self._url, rpm_repo, self._release)) + "/" def deploy_deb(self, packages: Packages): for package in packages.deb: path = packages.path(package) dist = self._release comp = "main" arch = packages.arch(package) logging.info( f"Deploy {path} distribution={dist};component={comp};" f"architecture={arch} to artifactory" ) self.deb(package).deploy_deb(path, dist, comp, arch) def deploy_rpm(self, packages: Packages): for package in packages.rpm: path = packages.path(package) logging.info(f"Deploy {path} to artifactory") self.rpm(package).deploy_file(path) def __path_helper(self, name, package) -> ArtifactorySaaSPath: url = "/".join((getattr(self, name + "_url"), package)) path = None if JFROG_API_KEY: path = ArtifactorySaaSPath(url, apikey=JFROG_API_KEY) elif JFROG_TOKEN: path = ArtifactorySaaSPath(url, token=JFROG_TOKEN) else: raise KeyError("Neither JFROG_API_KEY nor JFROG_TOKEN env are defined") return path def deb(self, package) -> ArtifactorySaaSPath: return self.__path_helper("_deb", package) def rpm(self, package) -> ArtifactorySaaSPath: return self.__path_helper("_rpm", package) def commit(name): r = re.compile(r"^([0-9]|[a-f]){40}$") if not r.match(name): raise argparse.ArgumentTypeError( "commit hash should contain exactly 40 hex characters" ) return name def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description="Program to download artifacts from S3 and push them to " "artifactory. ENV variables JFROG_API_KEY and JFROG_TOKEN are used " "for authentication in the given order", ) parser.add_argument("--pull-request", type=int, default=0, help="release name") parser.add_argument( "--commit", required=True, type=commit, help="commit hash for S3 bucket" ) parser.add_argument("--release", required=True, type=Release(), help="release name") parser.add_argument( "--bucket-name", default="clickhouse-builds", help="AWS S3 bucket name", ) parser.add_argument( "--check-name", default="ClickHouse build check (actions)", help="check name, a part of bucket path, " "will be converted to lower case with spaces->underscore", ) parser.add_argument( "--build-name", default="clang-13_relwithdebuginfo_none_bundled_" "unsplitted_notidy_without_coverage_deb", help="build name, a part of bucket path", ) parser.add_argument( "--deb", action="store_true", help="if Debian packages should be processed" ) parser.add_argument( "--rpm", action="store_true", help="if RPM packages should be processed" ) parser.add_argument( "--artifactory-url", default="https://clickhousedb.jfrog.io/artifactory" ) args = parser.parse_args() if not args.deb and not args.rpm: parser.error("at least one of --deb and --rpm should be specified") args.check_name = args.check_name.lower().replace(" ", "_") return args def main(): logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") args = parse_args() s3 = S3( args.bucket_name, args.pull_request, args.commit, args.check_name, args.build_name, args.release.version, ) art_client = Artifactory(args.artifactory_url, args.release.type) if args.deb: s3.download_deb() art_client.deploy_deb(s3.packages) if args.rpm: s3.download_rpm() art_client.deploy_rpm(s3.packages) if __name__ == "__main__": main()