ClickHouse/docker/packager/packager

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import subprocess
import os
import argparse
import logging
import sys

SCRIPT_PATH = os.path.realpath(__file__)


def check_image_exists_locally(image_name):
    try:
        output = subprocess.check_output(
            f"docker images -q {image_name} 2> /dev/null", shell=True
        )
        return output != ""
    except subprocess.CalledProcessError:
        return False


def pull_image(image_name):
    try:
        subprocess.check_call(f"docker pull {image_name}", shell=True)
        return True
    except subprocess.CalledProcessError:
        logging.info(f"Cannot pull image {image_name}".format())
        return False


def build_image(image_name, filepath):
    context = os.path.dirname(filepath)
    build_cmd = f"docker build --network=host -t {image_name} -f {filepath} {context}"
    logging.info("Will build image with cmd: '%s'", build_cmd)
    subprocess.check_call(
        build_cmd,
        shell=True,
    )


def run_docker_image_with_env(
    image_name, output, env_variables, ch_root, ccache_dir, docker_image_version
):
    env_part = " -e ".join(env_variables)
    if env_part:
        env_part = " -e " + env_part

    if sys.stdout.isatty():
        interactive = "-it"
    else:
        interactive = ""

    cmd = (
        f"docker run --network=host --rm --volume={output}:/output "
        f"--volume={ch_root}:/build --volume={ccache_dir}:/ccache {env_part} "
        f"{interactive} {image_name}:{docker_image_version}"
    )

    logging.info("Will build ClickHouse pkg with cmd: '%s'", cmd)

    subprocess.check_call(cmd, shell=True)


def is_release_build(build_type, package_type, sanitizer, split_binary):
    return (
        build_type == ""
        and package_type == "deb"
        and sanitizer == ""
        and not split_binary
    )


def parse_env_variables(
    build_type,
    compiler,
    sanitizer,
    package_type,
    image_type,
    cache,
    distcc_hosts,
    split_binary,
    clang_tidy,
    version,
    author,
    official,
    additional_pkgs,
    with_coverage,
    with_binaries,
):
    DARWIN_SUFFIX = "-darwin"
    DARWIN_ARM_SUFFIX = "-darwin-aarch64"
    ARM_SUFFIX = "-aarch64"
    FREEBSD_SUFFIX = "-freebsd"
    PPC_SUFFIX = "-ppc64le"

    result = []
    result.append("OUTPUT_DIR=/output")
    cmake_flags = ["$CMAKE_FLAGS"]

    is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
    is_cross_darwin_arm = compiler.endswith(DARWIN_ARM_SUFFIX)
    is_cross_arm = compiler.endswith(ARM_SUFFIX)
    is_cross_ppc = compiler.endswith(PPC_SUFFIX)
    is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)

    if is_cross_darwin:
        cc = compiler[: -len(DARWIN_SUFFIX)]
        cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/x86_64-apple-darwin-ar")
        cmake_flags.append(
            "-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/"
            "x86_64-apple-darwin-install_name_tool"
        )
        cmake_flags.append(
            "-DCMAKE_RANLIB:FILEPATH=/cctools/bin/x86_64-apple-darwin-ranlib"
        )
        cmake_flags.append("-DLINKER_NAME=/cctools/bin/x86_64-apple-darwin-ld")
        cmake_flags.append(
            "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-x86_64.cmake"
        )
    elif is_cross_darwin_arm:
        cc = compiler[: -len(DARWIN_ARM_SUFFIX)]
        cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/aarch64-apple-darwin-ar")
        cmake_flags.append(
            "-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/"
            "aarch64-apple-darwin-install_name_tool"
        )
        cmake_flags.append(
            "-DCMAKE_RANLIB:FILEPATH=/cctools/bin/aarch64-apple-darwin-ranlib"
        )
        cmake_flags.append("-DLINKER_NAME=/cctools/bin/aarch64-apple-darwin-ld")
        cmake_flags.append(
            "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-aarch64.cmake"
        )
    elif is_cross_arm:
        cc = compiler[: -len(ARM_SUFFIX)]
        cmake_flags.append(
            "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake"
        )
        result.append("DEB_ARCH=arm64")
    elif is_cross_freebsd:
        cc = compiler[: -len(FREEBSD_SUFFIX)]
        cmake_flags.append(
            "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake"
        )
    elif is_cross_ppc:
        cc = compiler[: -len(PPC_SUFFIX)]
        cmake_flags.append(
            "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-ppc64le.cmake"
        )
    else:
        cc = compiler
        result.append("DEB_ARCH=amd64")

    cxx = cc.replace("gcc", "g++").replace("clang", "clang++")

    if image_type == "deb":
        result.append("MAKE_DEB=true")
        cmake_flags.append("-DENABLE_TESTS=0")
        cmake_flags.append("-DENABLE_UTILS=0")
        cmake_flags.append("-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON")
        cmake_flags.append("-DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON")
        cmake_flags.append("-DCMAKE_AUTOGEN_VERBOSE=ON")
        cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr")
        cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc")
        cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var")
        if is_release_build(build_type, package_type, sanitizer, split_binary):
            cmake_flags.append("-DINSTALL_STRIPPED_BINARIES=ON")
            if is_cross_arm:
                cmake_flags.append("-DBUILD_STANDALONE_KEEPER=1")
            else:
                result.append("BUILD_MUSL_KEEPER=1")

    result.append(f"CC={cc}")
    result.append(f"CXX={cxx}")
    cmake_flags.append(f"-DCMAKE_C_COMPILER={cc}")
    cmake_flags.append(f"-DCMAKE_CXX_COMPILER={cxx}")

    # Create combined output archive for split build and for performance tests.
    if package_type == "performance":
        result.append("COMBINED_OUTPUT=performance")
        cmake_flags.append("-DENABLE_TESTS=0")
    elif package_type == "coverity":
        result.append("COMBINED_OUTPUT=coverity")
        result.append('COVERITY_TOKEN="$COVERITY_TOKEN"')
    elif split_binary:
        result.append("COMBINED_OUTPUT=shared_build")

    if sanitizer:
        result.append(f"SANITIZER={sanitizer}")
    if build_type:
        result.append(f"BUILD_TYPE={build_type.capitalize()}")
    else:
        result.append("BUILD_TYPE=None")

    if cache == "distcc":
        result.append(f"CCACHE_PREFIX={cache}")

    if cache:
        result.append("CCACHE_DIR=/ccache")
        result.append("CCACHE_BASEDIR=/build")
        result.append("CCACHE_NOHASHDIR=true")
        result.append("CCACHE_COMPILERCHECK=content")
        result.append("CCACHE_MAXSIZE=15G")
        # result.append("CCACHE_UMASK=777")

    if distcc_hosts:
        hosts_with_params = [f"{host}/24,lzo" for host in distcc_hosts] + [
            "localhost/`nproc`"
        ]
        result.append('DISTCC_HOSTS="' + " ".join(hosts_with_params) + '"')
    elif cache == "distcc":
        result.append('DISTCC_HOSTS="localhost/`nproc`"')

    if additional_pkgs:
        result.append("MAKE_APK=true")
        result.append("MAKE_RPM=true")
        result.append("MAKE_TGZ=true")

    if with_binaries == "programs":
        result.append("BINARY_OUTPUT=programs")
    elif with_binaries == "tests":
        result.append("ENABLE_TESTS=1")
        result.append("BINARY_OUTPUT=tests")
        cmake_flags.append("-DENABLE_TESTS=1")

    if split_binary:
        cmake_flags.append(
            "-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 "
            "-DCLICKHOUSE_SPLIT_BINARY=1"
        )
        # We can't always build utils because it requires too much space, but
        # we have to build them at least in some way in CI. The split build is
        # probably the least heavy disk-wise.
        cmake_flags.append("-DENABLE_UTILS=1")

    if clang_tidy:
        cmake_flags.append("-DENABLE_CLANG_TIDY=1")
        cmake_flags.append("-DENABLE_UTILS=1")
        cmake_flags.append("-DENABLE_TESTS=1")
        cmake_flags.append("-DENABLE_EXAMPLES=1")
        # Don't stop on first error to find more clang-tidy errors in one run.
        result.append("NINJA_FLAGS=-k0")

    if with_coverage:
        cmake_flags.append("-DWITH_COVERAGE=1")

    if version:
        result.append(f"VERSION_STRING='{version}'")

    if author:
        result.append(f"AUTHOR='{author}'")

    if official:
        cmake_flags.append("-DCLICKHOUSE_OFFICIAL_BUILD=1")

    result.append('CMAKE_FLAGS="' + " ".join(cmake_flags) + '"')

    return result


if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description="ClickHouse building script using prebuilt Docker image",
    )
    # 'performance' creates a combined .tgz with server
    # and configs to be used for performance test.
    parser.add_argument(
        "--package-type",
        choices=["deb", "binary", "performance", "coverity"],
        required=True,
    )
    parser.add_argument(
        "--clickhouse-repo-path",
        default=os.path.join(
            os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir
        ),
        help="ClickHouse git repository",
    )
    parser.add_argument("--output-dir", required=True)
    parser.add_argument("--build-type", choices=("debug", ""), default="")
    parser.add_argument(
        "--compiler",
        choices=(
            "clang-11",
            "clang-11-darwin",
            "clang-11-darwin-aarch64",
            "clang-11-aarch64",
            "clang-12",
            "clang-12-darwin",
            "clang-12-darwin-aarch64",
            "clang-12-aarch64",
            "clang-13",
            "clang-13-darwin",
            "clang-13-darwin-aarch64",
            "clang-13-aarch64",
            "clang-13-ppc64le",
            "clang-11-freebsd",
            "clang-12-freebsd",
            "clang-13-freebsd",
            "gcc-11",
        ),
        default="clang-13",
        help="a compiler to use",
    )
    parser.add_argument(
        "--sanitizer",
        choices=("address", "thread", "memory", "undefined", ""),
        default="",
    )
    parser.add_argument("--split-binary", action="store_true")
    parser.add_argument("--clang-tidy", action="store_true")
    parser.add_argument("--cache", choices=("ccache", "distcc", ""), default="")
    parser.add_argument(
        "--ccache_dir",
        default=os.getenv("HOME", "") + "/.ccache",
        help="a directory with ccache",
    )
    parser.add_argument("--distcc-hosts", nargs="+")
    parser.add_argument("--force-build-image", action="store_true")
    parser.add_argument("--version")
    parser.add_argument("--author", default="clickhouse", help="a package author")
    parser.add_argument("--official", action="store_true")
    parser.add_argument("--additional-pkgs", action="store_true")
    parser.add_argument("--with-coverage", action="store_true")
    parser.add_argument(
        "--with-binaries", choices=("programs", "tests", ""), default=""
    )
    parser.add_argument(
        "--docker-image-version", default="latest", help="docker image tag to use"
    )

    args = parser.parse_args()
    if not os.path.isabs(args.output_dir):
        args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir))

    image_type = (
        "binary"
        if args.package_type in ("performance", "coverity")
        else args.package_type
    )
    image_name = "clickhouse/binary-builder"

    if not os.path.isabs(args.clickhouse_repo_path):
        ch_root = os.path.abspath(os.path.join(os.getcwd(), args.clickhouse_repo_path))
    else:
        ch_root = args.clickhouse_repo_path

    if args.additional_pkgs and image_type != "deb":
        raise Exception("Can build additional packages only in deb build")

    if args.with_binaries != "" and image_type != "deb":
        raise Exception("Can add additional binaries only in deb build")

    if args.with_binaries != "" and image_type == "deb":
        logging.info("Should place %s to output", args.with_binaries)

    dockerfile = os.path.join(ch_root, "docker/packager", image_type, "Dockerfile")
    image_with_version = image_name + ":" + args.docker_image_version
    if (
        image_type != "freebsd"
        and not check_image_exists_locally(image_name)
        or args.force_build_image
    ):
        if not pull_image(image_with_version) or args.force_build_image:
            build_image(image_with_version, dockerfile)
    env_prepared = parse_env_variables(
        args.build_type,
        args.compiler,
        args.sanitizer,
        args.package_type,
        image_type,
        args.cache,
        args.distcc_hosts,
        args.split_binary,
        args.clang_tidy,
        args.version,
        args.author,
        args.official,
        args.additional_pkgs,
        args.with_coverage,
        args.with_binaries,
    )

    run_docker_image_with_env(
        image_name,
        args.output_dir,
        env_prepared,
        ch_root,
        args.ccache_dir,
        args.docker_image_version,
    )
    logging.info("Output placed into %s", args.output_dir)