mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
bb358617e1
The previous name was slightly misleading, e.g. it is not about "intalling stripped binaries" but about splitting debug symbols from the binary.
417 lines
14 KiB
Python
Executable File
417 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
import subprocess
|
|
import os
|
|
import argparse
|
|
import logging
|
|
import sys
|
|
from typing import List
|
|
|
|
SCRIPT_PATH = os.path.realpath(__file__)
|
|
IMAGE_TYPE = "binary"
|
|
|
|
|
|
def check_image_exists_locally(image_name):
|
|
try:
|
|
output = subprocess.check_output(
|
|
f"docker images -q {image_name} 2> /dev/null", shell=True
|
|
)
|
|
return output != ""
|
|
except subprocess.CalledProcessError:
|
|
return False
|
|
|
|
|
|
def pull_image(image_name):
|
|
try:
|
|
subprocess.check_call(f"docker pull {image_name}", shell=True)
|
|
return True
|
|
except subprocess.CalledProcessError:
|
|
logging.info(f"Cannot pull image {image_name}".format())
|
|
return False
|
|
|
|
|
|
def build_image(image_name, filepath):
|
|
context = os.path.dirname(filepath)
|
|
build_cmd = f"docker build --network=host -t {image_name} -f {filepath} {context}"
|
|
logging.info("Will build image with cmd: '%s'", build_cmd)
|
|
subprocess.check_call(
|
|
build_cmd,
|
|
shell=True,
|
|
)
|
|
|
|
|
|
def pre_build(repo_path: str, env_variables: List[str]):
|
|
if "WITH_PERFORMANCE=1" in env_variables:
|
|
current_branch = subprocess.check_output(
|
|
"git branch --show-current", shell=True, encoding="utf-8"
|
|
).strip()
|
|
is_shallow = (
|
|
subprocess.check_output(
|
|
"git rev-parse --is-shallow-repository", shell=True, encoding="utf-8"
|
|
)
|
|
== "true\n"
|
|
)
|
|
if is_shallow:
|
|
# I've spent quite some time on looking around the problem, and my
|
|
# conclusion is: in the current state the easiest way to go is to force
|
|
# unshallow repository for performance artifacts.
|
|
# To change it we need to rework our performance tests docker image
|
|
raise Exception("shallow repository is not suitable for performance builds")
|
|
if current_branch != "master":
|
|
cmd = (
|
|
f"git -C {repo_path} fetch --no-recurse-submodules "
|
|
"--no-tags origin master:master"
|
|
)
|
|
logging.info("Getting master branch for performance artifact: ''%s'", cmd)
|
|
subprocess.check_call(cmd, shell=True)
|
|
|
|
|
|
def run_docker_image_with_env(
|
|
image_name,
|
|
as_root,
|
|
output,
|
|
env_variables,
|
|
ch_root,
|
|
ccache_dir,
|
|
docker_image_version,
|
|
):
|
|
env_part = " -e ".join(env_variables)
|
|
if env_part:
|
|
env_part = " -e " + env_part
|
|
|
|
if sys.stdout.isatty():
|
|
interactive = "-it"
|
|
else:
|
|
interactive = ""
|
|
|
|
if as_root:
|
|
user = "0:0"
|
|
else:
|
|
user = f"{os.geteuid()}:{os.getegid()}"
|
|
|
|
cmd = (
|
|
f"docker run --network=host --user={user} --rm --volume={output}:/output "
|
|
f"--volume={ch_root}:/build --volume={ccache_dir}:/ccache {env_part} "
|
|
f"{interactive} {image_name}:{docker_image_version}"
|
|
)
|
|
|
|
logging.info("Will build ClickHouse pkg with cmd: '%s'", cmd)
|
|
|
|
subprocess.check_call(cmd, shell=True)
|
|
|
|
|
|
def is_release_build(build_type, package_type, sanitizer, split_binary):
|
|
return (
|
|
build_type == ""
|
|
and package_type == "deb"
|
|
and sanitizer == ""
|
|
and not split_binary
|
|
)
|
|
|
|
|
|
def parse_env_variables(
|
|
build_type,
|
|
compiler,
|
|
sanitizer,
|
|
package_type,
|
|
cache,
|
|
distcc_hosts,
|
|
split_binary,
|
|
clang_tidy,
|
|
version,
|
|
author,
|
|
official,
|
|
additional_pkgs,
|
|
with_coverage,
|
|
with_binaries,
|
|
):
|
|
DARWIN_SUFFIX = "-darwin"
|
|
DARWIN_ARM_SUFFIX = "-darwin-aarch64"
|
|
ARM_SUFFIX = "-aarch64"
|
|
FREEBSD_SUFFIX = "-freebsd"
|
|
PPC_SUFFIX = "-ppc64le"
|
|
|
|
result = []
|
|
result.append("OUTPUT_DIR=/output")
|
|
cmake_flags = ["$CMAKE_FLAGS"]
|
|
|
|
is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
|
|
is_cross_darwin_arm = compiler.endswith(DARWIN_ARM_SUFFIX)
|
|
is_cross_arm = compiler.endswith(ARM_SUFFIX)
|
|
is_cross_ppc = compiler.endswith(PPC_SUFFIX)
|
|
is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
|
|
|
|
if is_cross_darwin:
|
|
cc = compiler[: -len(DARWIN_SUFFIX)]
|
|
cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/x86_64-apple-darwin-ar")
|
|
cmake_flags.append(
|
|
"-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/"
|
|
"x86_64-apple-darwin-install_name_tool"
|
|
)
|
|
cmake_flags.append(
|
|
"-DCMAKE_RANLIB:FILEPATH=/cctools/bin/x86_64-apple-darwin-ranlib"
|
|
)
|
|
cmake_flags.append("-DLINKER_NAME=/cctools/bin/x86_64-apple-darwin-ld")
|
|
cmake_flags.append(
|
|
"-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-x86_64.cmake"
|
|
)
|
|
elif is_cross_darwin_arm:
|
|
cc = compiler[: -len(DARWIN_ARM_SUFFIX)]
|
|
cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/aarch64-apple-darwin-ar")
|
|
cmake_flags.append(
|
|
"-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/"
|
|
"aarch64-apple-darwin-install_name_tool"
|
|
)
|
|
cmake_flags.append(
|
|
"-DCMAKE_RANLIB:FILEPATH=/cctools/bin/aarch64-apple-darwin-ranlib"
|
|
)
|
|
cmake_flags.append("-DLINKER_NAME=/cctools/bin/aarch64-apple-darwin-ld")
|
|
cmake_flags.append(
|
|
"-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-aarch64.cmake"
|
|
)
|
|
elif is_cross_arm:
|
|
cc = compiler[: -len(ARM_SUFFIX)]
|
|
cmake_flags.append(
|
|
"-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake"
|
|
)
|
|
result.append("DEB_ARCH=arm64")
|
|
elif is_cross_freebsd:
|
|
cc = compiler[: -len(FREEBSD_SUFFIX)]
|
|
cmake_flags.append(
|
|
"-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake"
|
|
)
|
|
elif is_cross_ppc:
|
|
cc = compiler[: -len(PPC_SUFFIX)]
|
|
cmake_flags.append(
|
|
"-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-ppc64le.cmake"
|
|
)
|
|
else:
|
|
cc = compiler
|
|
result.append("DEB_ARCH=amd64")
|
|
|
|
cxx = cc.replace("gcc", "g++").replace("clang", "clang++")
|
|
|
|
if package_type == "deb":
|
|
result.append("MAKE_DEB=true")
|
|
cmake_flags.append("-DENABLE_TESTS=0")
|
|
cmake_flags.append("-DENABLE_UTILS=0")
|
|
cmake_flags.append("-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON")
|
|
cmake_flags.append("-DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON")
|
|
cmake_flags.append("-DCMAKE_AUTOGEN_VERBOSE=ON")
|
|
cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr")
|
|
cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc")
|
|
cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var")
|
|
if is_release_build(build_type, package_type, sanitizer, split_binary):
|
|
cmake_flags.append("-DSPLIT_DEBUG_SYMBOLS=ON")
|
|
result.append("WITH_PERFORMANCE=1")
|
|
if is_cross_arm:
|
|
cmake_flags.append("-DBUILD_STANDALONE_KEEPER=1")
|
|
else:
|
|
result.append("BUILD_MUSL_KEEPER=1")
|
|
|
|
result.append(f"CC={cc}")
|
|
result.append(f"CXX={cxx}")
|
|
cmake_flags.append(f"-DCMAKE_C_COMPILER={cc}")
|
|
cmake_flags.append(f"-DCMAKE_CXX_COMPILER={cxx}")
|
|
|
|
# Create combined output archive for split build and for performance tests.
|
|
if package_type == "coverity":
|
|
result.append("COMBINED_OUTPUT=coverity")
|
|
result.append('COVERITY_TOKEN="$COVERITY_TOKEN"')
|
|
elif split_binary:
|
|
result.append("COMBINED_OUTPUT=shared_build")
|
|
|
|
if sanitizer:
|
|
result.append(f"SANITIZER={sanitizer}")
|
|
if build_type:
|
|
result.append(f"BUILD_TYPE={build_type.capitalize()}")
|
|
else:
|
|
result.append("BUILD_TYPE=None")
|
|
|
|
if cache == "distcc":
|
|
result.append(f"CCACHE_PREFIX={cache}")
|
|
|
|
if cache:
|
|
result.append("CCACHE_DIR=/ccache")
|
|
result.append("CCACHE_BASEDIR=/build")
|
|
result.append("CCACHE_NOHASHDIR=true")
|
|
result.append("CCACHE_COMPILERCHECK=content")
|
|
result.append("CCACHE_MAXSIZE=15G")
|
|
# result.append("CCACHE_UMASK=777")
|
|
|
|
if distcc_hosts:
|
|
hosts_with_params = [f"{host}/24,lzo" for host in distcc_hosts] + [
|
|
"localhost/`nproc`"
|
|
]
|
|
result.append('DISTCC_HOSTS="' + " ".join(hosts_with_params) + '"')
|
|
elif cache == "distcc":
|
|
result.append('DISTCC_HOSTS="localhost/`nproc`"')
|
|
|
|
if additional_pkgs:
|
|
result.append("MAKE_APK=true")
|
|
result.append("MAKE_RPM=true")
|
|
result.append("MAKE_TGZ=true")
|
|
|
|
if with_binaries == "programs":
|
|
result.append("BINARY_OUTPUT=programs")
|
|
elif with_binaries == "tests":
|
|
result.append("ENABLE_TESTS=1")
|
|
result.append("BINARY_OUTPUT=tests")
|
|
cmake_flags.append("-DENABLE_TESTS=1")
|
|
|
|
if split_binary:
|
|
cmake_flags.append(
|
|
"-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 "
|
|
"-DCLICKHOUSE_SPLIT_BINARY=1"
|
|
)
|
|
# We can't always build utils because it requires too much space, but
|
|
# we have to build them at least in some way in CI. The split build is
|
|
# probably the least heavy disk-wise.
|
|
cmake_flags.append("-DENABLE_UTILS=1")
|
|
|
|
if clang_tidy:
|
|
cmake_flags.append("-DENABLE_CLANG_TIDY=1")
|
|
cmake_flags.append("-DENABLE_UTILS=1")
|
|
cmake_flags.append("-DENABLE_TESTS=1")
|
|
cmake_flags.append("-DENABLE_EXAMPLES=1")
|
|
# Don't stop on first error to find more clang-tidy errors in one run.
|
|
result.append("NINJA_FLAGS=-k0")
|
|
|
|
if with_coverage:
|
|
cmake_flags.append("-DWITH_COVERAGE=1")
|
|
|
|
if version:
|
|
result.append(f"VERSION_STRING='{version}'")
|
|
|
|
if author:
|
|
result.append(f"AUTHOR='{author}'")
|
|
|
|
if official:
|
|
cmake_flags.append("-DCLICKHOUSE_OFFICIAL_BUILD=1")
|
|
|
|
result.append('CMAKE_FLAGS="' + " ".join(cmake_flags) + '"')
|
|
|
|
return result
|
|
|
|
|
|
def dir_name(name: str) -> str:
|
|
if not os.path.isabs(name):
|
|
name = os.path.abspath(os.path.join(os.getcwd(), name))
|
|
return name
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
|
|
parser = argparse.ArgumentParser(
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
description="ClickHouse building script using prebuilt Docker image",
|
|
)
|
|
parser.add_argument(
|
|
"--package-type",
|
|
choices=["deb", "binary", "coverity"],
|
|
required=True,
|
|
)
|
|
parser.add_argument(
|
|
"--clickhouse-repo-path",
|
|
default=os.path.join(os.path.dirname(SCRIPT_PATH), os.pardir, os.pardir),
|
|
type=dir_name,
|
|
help="ClickHouse git repository",
|
|
)
|
|
parser.add_argument("--output-dir", type=dir_name, required=True)
|
|
parser.add_argument("--build-type", choices=("debug", ""), default="")
|
|
|
|
parser.add_argument(
|
|
"--compiler",
|
|
choices=(
|
|
"clang-14",
|
|
"clang-14-darwin",
|
|
"clang-14-darwin-aarch64",
|
|
"clang-14-aarch64",
|
|
"clang-14-ppc64le",
|
|
"clang-14-freebsd",
|
|
"gcc-11",
|
|
),
|
|
default="clang-14",
|
|
help="a compiler to use",
|
|
)
|
|
parser.add_argument(
|
|
"--sanitizer",
|
|
choices=("address", "thread", "memory", "undefined", ""),
|
|
default="",
|
|
)
|
|
|
|
parser.add_argument("--split-binary", action="store_true")
|
|
parser.add_argument("--clang-tidy", action="store_true")
|
|
parser.add_argument("--cache", choices=("ccache", "distcc", ""), default="")
|
|
parser.add_argument(
|
|
"--ccache_dir",
|
|
default=os.getenv("HOME", "") + "/.ccache",
|
|
type=dir_name,
|
|
help="a directory with ccache",
|
|
)
|
|
parser.add_argument("--distcc-hosts", nargs="+")
|
|
parser.add_argument("--force-build-image", action="store_true")
|
|
parser.add_argument("--version")
|
|
parser.add_argument("--author", default="clickhouse", help="a package author")
|
|
parser.add_argument("--official", action="store_true")
|
|
parser.add_argument("--additional-pkgs", action="store_true")
|
|
parser.add_argument("--with-coverage", action="store_true")
|
|
parser.add_argument(
|
|
"--with-binaries", choices=("programs", "tests", ""), default=""
|
|
)
|
|
parser.add_argument(
|
|
"--docker-image-version", default="latest", help="docker image tag to use"
|
|
)
|
|
parser.add_argument(
|
|
"--as-root", action="store_true", help="if the container should run as root"
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
image_name = f"clickhouse/{IMAGE_TYPE}-builder"
|
|
|
|
ch_root = args.clickhouse_repo_path
|
|
|
|
if args.additional_pkgs and args.package_type != "deb":
|
|
raise Exception("Can build additional packages only in deb build")
|
|
|
|
if args.with_binaries != "" and args.package_type != "deb":
|
|
raise Exception("Can add additional binaries only in deb build")
|
|
|
|
if args.with_binaries != "" and args.package_type == "deb":
|
|
logging.info("Should place %s to output", args.with_binaries)
|
|
|
|
dockerfile = os.path.join(ch_root, "docker/packager", IMAGE_TYPE, "Dockerfile")
|
|
image_with_version = image_name + ":" + args.docker_image_version
|
|
if not check_image_exists_locally(image_name) or args.force_build_image:
|
|
if not pull_image(image_with_version) or args.force_build_image:
|
|
build_image(image_with_version, dockerfile)
|
|
env_prepared = parse_env_variables(
|
|
args.build_type,
|
|
args.compiler,
|
|
args.sanitizer,
|
|
args.package_type,
|
|
args.cache,
|
|
args.distcc_hosts,
|
|
args.split_binary,
|
|
args.clang_tidy,
|
|
args.version,
|
|
args.author,
|
|
args.official,
|
|
args.additional_pkgs,
|
|
args.with_coverage,
|
|
args.with_binaries,
|
|
)
|
|
|
|
pre_build(args.clickhouse_repo_path, env_prepared)
|
|
run_docker_image_with_env(
|
|
image_name,
|
|
args.as_root,
|
|
args.output_dir,
|
|
env_prepared,
|
|
ch_root,
|
|
args.ccache_dir,
|
|
args.docker_image_version,
|
|
)
|
|
logging.info("Output placed into %s", args.output_dir)
|