mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-12-05 05:52:05 +00:00
224 lines
9.4 KiB
Python
Executable File
224 lines
9.4 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#-*- coding: utf-8 -*-
|
|
import subprocess
|
|
import os
|
|
import argparse
|
|
import logging
|
|
import sys
|
|
|
|
SCRIPT_PATH = os.path.realpath(__file__)
|
|
|
|
IMAGE_MAP = {
|
|
"deb": "yandex/clickhouse-deb-builder",
|
|
"binary": "yandex/clickhouse-binary-builder",
|
|
}
|
|
|
|
def check_image_exists_locally(image_name):
|
|
try:
|
|
output = subprocess.check_output("docker images -q {} 2> /dev/null".format(image_name), shell=True)
|
|
return output != ""
|
|
except subprocess.CalledProcessError as ex:
|
|
return False
|
|
|
|
def pull_image(image_name):
|
|
try:
|
|
subprocess.check_call("docker pull {}".format(image_name), shell=True)
|
|
return True
|
|
except subprocess.CalledProcessError as ex:
|
|
logging.info("Cannot pull image {}".format(image_name))
|
|
return False
|
|
|
|
def build_image(image_name, filepath):
|
|
subprocess.check_call("docker build --network=host -t {} -f {} .".format(image_name, filepath), shell=True)
|
|
|
|
def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache_dir):
|
|
env_part = " -e ".join(env_variables)
|
|
if env_part:
|
|
env_part = " -e " + env_part
|
|
|
|
if sys.stdout.isatty():
|
|
interactive = "-it"
|
|
else:
|
|
interactive = ""
|
|
|
|
cmd = "docker run --network=host --rm --volume={output_path}:/output --volume={ch_root}:/build --volume={ccache_dir}:/ccache {env} {interactive} {img_name}".format(
|
|
output_path=output,
|
|
ch_root=ch_root,
|
|
ccache_dir=ccache_dir,
|
|
env=env_part,
|
|
img_name=image_name,
|
|
interactive=interactive
|
|
)
|
|
|
|
logging.info("Will build ClickHouse pkg with cmd: '{}'".format(cmd))
|
|
|
|
subprocess.check_call(cmd, shell=True)
|
|
|
|
def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, unbundled, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage, with_binaries):
|
|
CLANG_PREFIX = "clang"
|
|
DARWIN_SUFFIX = "-darwin"
|
|
ARM_SUFFIX = "-aarch64"
|
|
FREEBSD_SUFFIX = "-freebsd"
|
|
|
|
result = []
|
|
cmake_flags = ['$CMAKE_FLAGS', '-DADD_GDB_INDEX_FOR_GOLD=1']
|
|
|
|
is_clang = compiler.startswith(CLANG_PREFIX)
|
|
is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
|
|
is_cross_arm = compiler.endswith(ARM_SUFFIX)
|
|
is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
|
|
is_cross_compile = is_cross_darwin or is_cross_arm or is_cross_freebsd
|
|
|
|
# Explicitly use LLD with Clang by default.
|
|
# Don't force linker for cross-compilation.
|
|
if is_clang and not is_cross_compile:
|
|
cmake_flags.append("-DLINKER_NAME=lld")
|
|
|
|
if is_cross_darwin:
|
|
cc = compiler[:-len(DARWIN_SUFFIX)]
|
|
cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/x86_64-apple-darwin-ar")
|
|
cmake_flags.append("-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/x86_64-apple-darwin-install_name_tool")
|
|
cmake_flags.append("-DCMAKE_RANLIB:FILEPATH=/cctools/bin/x86_64-apple-darwin-ranlib")
|
|
cmake_flags.append("-DLINKER_NAME=/cctools/bin/x86_64-apple-darwin-ld")
|
|
cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-x86_64.cmake")
|
|
elif is_cross_arm:
|
|
cc = compiler[:-len(ARM_SUFFIX)]
|
|
cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake")
|
|
elif is_cross_freebsd:
|
|
cc = compiler[:-len(FREEBSD_SUFFIX)]
|
|
cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake")
|
|
else:
|
|
cc = compiler
|
|
|
|
cxx = cc.replace('gcc', 'g++').replace('clang', 'clang++')
|
|
|
|
if image_type == "deb":
|
|
result.append("DEB_CC={}".format(cc))
|
|
result.append("DEB_CXX={}".format(cxx))
|
|
elif image_type == "binary":
|
|
result.append("CC={}".format(cc))
|
|
result.append("CXX={}".format(cxx))
|
|
cmake_flags.append('-DCMAKE_C_COMPILER=`which {}`'.format(cc))
|
|
cmake_flags.append('-DCMAKE_CXX_COMPILER=`which {}`'.format(cxx))
|
|
|
|
# Create combined output archive for split build and for performance tests.
|
|
if package_type == "performance":
|
|
result.append("COMBINED_OUTPUT=performance")
|
|
elif split_binary:
|
|
result.append("COMBINED_OUTPUT=shared_build")
|
|
|
|
if sanitizer:
|
|
result.append("SANITIZER={}".format(sanitizer))
|
|
if build_type:
|
|
result.append("BUILD_TYPE={}".format(build_type))
|
|
|
|
if cache == 'distcc':
|
|
result.append("CCACHE_PREFIX={}".format(cache))
|
|
|
|
if cache:
|
|
result.append("CCACHE_DIR=/ccache")
|
|
result.append("CCACHE_BASEDIR=/build")
|
|
result.append("CCACHE_NOHASHDIR=true")
|
|
result.append("CCACHE_COMPILERCHECK=content")
|
|
result.append("CCACHE_MAXSIZE=15G")
|
|
# result.append("CCACHE_UMASK=777")
|
|
|
|
if distcc_hosts:
|
|
hosts_with_params = ["{}/24,lzo".format(host) for host in distcc_hosts] + ["localhost/`nproc`"]
|
|
result.append('DISTCC_HOSTS="{}"'.format(" ".join(hosts_with_params)))
|
|
elif cache == "distcc":
|
|
result.append('DISTCC_HOSTS="{}"'.format("localhost/`nproc`"))
|
|
|
|
if alien_pkgs:
|
|
result.append("ALIEN_PKGS='" + ' '.join(['--' + pkg for pkg in alien_pkgs]) + "'")
|
|
|
|
if with_binaries == "programs":
|
|
result.append('BINARY_OUTPUT=programs')
|
|
elif with_binaries == "tests":
|
|
result.append('ENABLE_TESTS=1')
|
|
result.append('BINARY_OUTPUT=tests')
|
|
cmake_flags.append('-DENABLE_TESTS=1')
|
|
cmake_flags.append('-DUSE_GTEST=1')
|
|
|
|
if unbundled:
|
|
# TODO: fix build with ENABLE_RDKAFKA
|
|
cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0')
|
|
|
|
if split_binary:
|
|
cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1')
|
|
|
|
if clang_tidy:
|
|
cmake_flags.append('-DENABLE_CLANG_TIDY=1')
|
|
|
|
if with_coverage:
|
|
cmake_flags.append('-DWITH_COVERAGE=1')
|
|
|
|
if version:
|
|
result.append("VERSION_STRING='{}'".format(version))
|
|
|
|
if author:
|
|
result.append("AUTHOR='{}'".format(author))
|
|
|
|
if official:
|
|
cmake_flags.append('-DYANDEX_OFFICIAL_BUILD=1')
|
|
|
|
result.append('CMAKE_FLAGS="' + ' '.join(cmake_flags) + '"')
|
|
|
|
return result
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
|
|
parser = argparse.ArgumentParser(description="ClickHouse building script using prebuilt Docker image")
|
|
# 'performance' creates a combined .tgz with server and configs to be used for performance test.
|
|
parser.add_argument("--package-type", choices=['deb', 'binary', 'performance'], required=True)
|
|
parser.add_argument("--clickhouse-repo-path", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir))
|
|
parser.add_argument("--output-dir", required=True)
|
|
parser.add_argument("--build-type", choices=("debug", ""), default="")
|
|
parser.add_argument("--compiler", choices=("clang-10-darwin", "clang-10-aarch64", "clang-10-freebsd", "gcc-9", "clang-10"), default="gcc-9")
|
|
parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="")
|
|
parser.add_argument("--unbundled", action="store_true")
|
|
parser.add_argument("--split-binary", action="store_true")
|
|
parser.add_argument("--clang-tidy", action="store_true")
|
|
parser.add_argument("--cache", choices=("", "ccache", "distcc"), default="")
|
|
parser.add_argument("--ccache_dir", default= os.getenv("HOME", "") + '/.ccache')
|
|
parser.add_argument("--distcc-hosts", nargs="+")
|
|
parser.add_argument("--force-build-image", action="store_true")
|
|
parser.add_argument("--version")
|
|
parser.add_argument("--author", default="clickhouse")
|
|
parser.add_argument("--official", action="store_true")
|
|
parser.add_argument("--alien-pkgs", nargs='+', default=[])
|
|
parser.add_argument("--with-coverage", action="store_true")
|
|
parser.add_argument("--with-binaries", choices=("programs", "tests", ""), default="")
|
|
|
|
args = parser.parse_args()
|
|
if not os.path.isabs(args.output_dir):
|
|
args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir))
|
|
|
|
image_type = 'binary' if args.package_type == 'performance' else args.package_type
|
|
image_name = IMAGE_MAP[image_type]
|
|
|
|
if not os.path.isabs(args.clickhouse_repo_path):
|
|
ch_root = os.path.abspath(os.path.join(os.getcwd(), args.clickhouse_repo_path))
|
|
else:
|
|
ch_root = args.clickhouse_repo_path
|
|
|
|
if args.alien_pkgs and not image_type == "deb":
|
|
raise Exception("Can add alien packages only in deb build")
|
|
|
|
if args.with_binaries != "" and not image_type == "deb":
|
|
raise Exception("Can add additional binaries only in deb build")
|
|
|
|
if args.with_binaries != "" and image_type == "deb":
|
|
logging.info("Should place {} to output".format(args.with_binaries))
|
|
|
|
dockerfile = os.path.join(ch_root, "docker/packager", image_type, "Dockerfile")
|
|
if image_type != "freebsd" and not check_image_exists_locally(image_name) or args.force_build_image:
|
|
if not pull_image(image_name) or args.force_build_image:
|
|
build_image(image_name, dockerfile)
|
|
env_prepared = parse_env_variables(
|
|
args.build_type, args.compiler, args.sanitizer, args.package_type, image_type,
|
|
args.cache, args.distcc_hosts, args.unbundled, args.split_binary, args.clang_tidy,
|
|
args.version, args.author, args.official, args.alien_pkgs, args.with_coverage, args.with_binaries)
|
|
run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir)
|
|
logging.info("Output placed into {}".format(args.output_dir))
|