mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-27 18:12:02 +00:00
448 lines
15 KiB
Python
Executable File
448 lines
15 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
import subprocess
|
|
import os
|
|
import argparse
|
|
import logging
|
|
import sys
|
|
from typing import List
|
|
|
|
SCRIPT_PATH = os.path.realpath(__file__)
|
|
IMAGE_TYPE = "binary"
|
|
|
|
|
|
def check_image_exists_locally(image_name):
|
|
try:
|
|
output = subprocess.check_output(
|
|
f"docker images -q {image_name} 2> /dev/null", shell=True
|
|
)
|
|
return output != ""
|
|
except subprocess.CalledProcessError:
|
|
return False
|
|
|
|
|
|
def pull_image(image_name):
|
|
try:
|
|
subprocess.check_call(f"docker pull {image_name}", shell=True)
|
|
return True
|
|
except subprocess.CalledProcessError:
|
|
logging.info(f"Cannot pull image {image_name}".format())
|
|
return False
|
|
|
|
|
|
def build_image(image_name, filepath):
|
|
context = os.path.dirname(filepath)
|
|
build_cmd = f"docker build --network=host -t {image_name} -f {filepath} {context}"
|
|
logging.info("Will build image with cmd: '%s'", build_cmd)
|
|
subprocess.check_call(
|
|
build_cmd,
|
|
shell=True,
|
|
)
|
|
|
|
|
|
def pre_build(repo_path: str, env_variables: List[str]):
|
|
if "WITH_PERFORMANCE=1" in env_variables:
|
|
current_branch = subprocess.check_output(
|
|
"git branch --show-current", shell=True, encoding="utf-8"
|
|
).strip()
|
|
is_shallow = (
|
|
subprocess.check_output(
|
|
"git rev-parse --is-shallow-repository", shell=True, encoding="utf-8"
|
|
)
|
|
== "true\n"
|
|
)
|
|
if is_shallow:
|
|
# I've spent quite some time on looking around the problem, and my
|
|
# conclusion is: in the current state the easiest way to go is to force
|
|
# unshallow repository for performance artifacts.
|
|
# To change it we need to rework our performance tests docker image
|
|
raise Exception("shallow repository is not suitable for performance builds")
|
|
if current_branch != "master":
|
|
cmd = (
|
|
f"git -C {repo_path} fetch --no-recurse-submodules "
|
|
"--no-tags origin master:master"
|
|
)
|
|
logging.info("Getting master branch for performance artifact: '%s'", cmd)
|
|
subprocess.check_call(cmd, shell=True)
|
|
|
|
|
|
def run_docker_image_with_env(
|
|
image_name,
|
|
as_root,
|
|
output,
|
|
env_variables,
|
|
ch_root,
|
|
ccache_dir,
|
|
docker_image_version,
|
|
):
|
|
env_part = " -e ".join(env_variables)
|
|
if env_part:
|
|
env_part = " -e " + env_part
|
|
|
|
if sys.stdout.isatty():
|
|
interactive = "-it"
|
|
else:
|
|
interactive = ""
|
|
|
|
if as_root:
|
|
user = "0:0"
|
|
else:
|
|
user = f"{os.geteuid()}:{os.getegid()}"
|
|
|
|
cmd = (
|
|
f"docker run --network=host --user={user} --rm --volume={output}:/output "
|
|
f"--volume={ch_root}:/build --volume={ccache_dir}:/ccache {env_part} "
|
|
f"{interactive} {image_name}:{docker_image_version}"
|
|
)
|
|
|
|
logging.info("Will build ClickHouse pkg with cmd: '%s'", cmd)
|
|
|
|
subprocess.check_call(cmd, shell=True)
|
|
|
|
|
|
def is_release_build(build_type, package_type, sanitizer, shared_libraries):
|
|
return (
|
|
build_type == ""
|
|
and package_type == "deb"
|
|
and sanitizer == ""
|
|
and not shared_libraries
|
|
)
|
|
|
|
|
|
def parse_env_variables(
|
|
build_type,
|
|
compiler,
|
|
sanitizer,
|
|
package_type,
|
|
cache,
|
|
distcc_hosts,
|
|
shared_libraries,
|
|
clang_tidy,
|
|
version,
|
|
author,
|
|
official,
|
|
additional_pkgs,
|
|
with_coverage,
|
|
with_binaries,
|
|
):
|
|
DARWIN_SUFFIX = "-darwin"
|
|
DARWIN_ARM_SUFFIX = "-darwin-aarch64"
|
|
ARM_SUFFIX = "-aarch64"
|
|
ARM_V80COMPAT_SUFFIX = "-aarch64-v80compat"
|
|
FREEBSD_SUFFIX = "-freebsd"
|
|
PPC_SUFFIX = "-ppc64le"
|
|
AMD64_COMPAT_SUFFIX = "-amd64-compat"
|
|
|
|
result = []
|
|
result.append("OUTPUT_DIR=/output")
|
|
cmake_flags = ["$CMAKE_FLAGS"]
|
|
build_target = "clickhouse-bundle"
|
|
|
|
is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
|
|
is_cross_darwin_arm = compiler.endswith(DARWIN_ARM_SUFFIX)
|
|
is_cross_arm = compiler.endswith(ARM_SUFFIX)
|
|
is_cross_arm_v80compat = compiler.endswith(ARM_V80COMPAT_SUFFIX)
|
|
is_cross_ppc = compiler.endswith(PPC_SUFFIX)
|
|
is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
|
|
is_amd64_compat = compiler.endswith(AMD64_COMPAT_SUFFIX)
|
|
|
|
if is_cross_darwin:
|
|
cc = compiler[: -len(DARWIN_SUFFIX)]
|
|
cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/x86_64-apple-darwin-ar")
|
|
cmake_flags.append(
|
|
"-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/"
|
|
"x86_64-apple-darwin-install_name_tool"
|
|
)
|
|
cmake_flags.append(
|
|
"-DCMAKE_RANLIB:FILEPATH=/cctools/bin/x86_64-apple-darwin-ranlib"
|
|
)
|
|
cmake_flags.append("-DLINKER_NAME=/cctools/bin/x86_64-apple-darwin-ld")
|
|
cmake_flags.append(
|
|
"-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-x86_64.cmake"
|
|
)
|
|
elif is_cross_darwin_arm:
|
|
cc = compiler[: -len(DARWIN_ARM_SUFFIX)]
|
|
cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/aarch64-apple-darwin-ar")
|
|
cmake_flags.append(
|
|
"-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/"
|
|
"aarch64-apple-darwin-install_name_tool"
|
|
)
|
|
cmake_flags.append(
|
|
"-DCMAKE_RANLIB:FILEPATH=/cctools/bin/aarch64-apple-darwin-ranlib"
|
|
)
|
|
cmake_flags.append("-DLINKER_NAME=/cctools/bin/aarch64-apple-darwin-ld")
|
|
cmake_flags.append(
|
|
"-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-aarch64.cmake"
|
|
)
|
|
elif is_cross_arm:
|
|
cc = compiler[: -len(ARM_SUFFIX)]
|
|
cmake_flags.append(
|
|
"-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake"
|
|
)
|
|
result.append("DEB_ARCH=arm64")
|
|
elif is_cross_arm_v80compat:
|
|
cc = compiler[: -len(ARM_V80COMPAT_SUFFIX)]
|
|
cmake_flags.append(
|
|
"-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake"
|
|
)
|
|
cmake_flags.append("-DNO_ARMV81_OR_HIGHER=1")
|
|
result.append("DEB_ARCH=arm64")
|
|
elif is_cross_freebsd:
|
|
cc = compiler[: -len(FREEBSD_SUFFIX)]
|
|
cmake_flags.append(
|
|
"-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake"
|
|
)
|
|
elif is_cross_ppc:
|
|
cc = compiler[: -len(PPC_SUFFIX)]
|
|
cmake_flags.append(
|
|
"-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-ppc64le.cmake"
|
|
)
|
|
elif is_amd64_compat:
|
|
cc = compiler[: -len(AMD64_COMPAT_SUFFIX)]
|
|
result.append("DEB_ARCH=amd64")
|
|
cmake_flags.append("-DNO_SSE3_OR_HIGHER=1")
|
|
else:
|
|
cc = compiler
|
|
result.append("DEB_ARCH=amd64")
|
|
|
|
cxx = cc.replace("gcc", "g++").replace("clang", "clang++")
|
|
|
|
if package_type == "deb":
|
|
# NOTE: This are the env for packages/build script
|
|
result.append("MAKE_DEB=true")
|
|
cmake_flags.append("-DENABLE_TESTS=0")
|
|
cmake_flags.append("-DENABLE_UTILS=0")
|
|
cmake_flags.append("-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON")
|
|
cmake_flags.append("-DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON")
|
|
cmake_flags.append("-DCMAKE_AUTOGEN_VERBOSE=ON")
|
|
cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr")
|
|
cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc")
|
|
cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var")
|
|
if is_release_build(build_type, package_type, sanitizer, shared_libraries):
|
|
cmake_flags.append("-DSPLIT_DEBUG_SYMBOLS=ON")
|
|
result.append("WITH_PERFORMANCE=1")
|
|
if is_cross_arm:
|
|
cmake_flags.append("-DBUILD_STANDALONE_KEEPER=1")
|
|
else:
|
|
result.append("BUILD_MUSL_KEEPER=1")
|
|
|
|
result.append(f"CC={cc}")
|
|
result.append(f"CXX={cxx}")
|
|
cmake_flags.append(f"-DCMAKE_C_COMPILER={cc}")
|
|
cmake_flags.append(f"-DCMAKE_CXX_COMPILER={cxx}")
|
|
|
|
# Create combined output archive for shared library build and for performance tests.
|
|
if package_type == "coverity":
|
|
result.append("COMBINED_OUTPUT=coverity")
|
|
result.append('COVERITY_TOKEN="$COVERITY_TOKEN"')
|
|
elif shared_libraries:
|
|
result.append("COMBINED_OUTPUT=shared_build")
|
|
|
|
if sanitizer:
|
|
result.append(f"SANITIZER={sanitizer}")
|
|
if build_type:
|
|
result.append(f"BUILD_TYPE={build_type.capitalize()}")
|
|
else:
|
|
result.append("BUILD_TYPE=None")
|
|
|
|
if cache == "distcc":
|
|
result.append(f"CCACHE_PREFIX={cache}")
|
|
|
|
if cache:
|
|
result.append("CCACHE_DIR=/ccache")
|
|
result.append("CCACHE_COMPRESSLEVEL=5")
|
|
result.append("CCACHE_BASEDIR=/build")
|
|
result.append("CCACHE_NOHASHDIR=true")
|
|
result.append("CCACHE_COMPILERCHECK=content")
|
|
cache_maxsize = "15G"
|
|
if clang_tidy:
|
|
# 15G is not enough for tidy build
|
|
cache_maxsize = "25G"
|
|
|
|
# `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above.
|
|
# It's there to have the clang-tidy cache embedded into our standard `CCACHE_DIR`
|
|
result.append("CTCACHE_DIR=/ccache/clang-tidy-cache")
|
|
result.append(f"CCACHE_MAXSIZE={cache_maxsize}")
|
|
|
|
if distcc_hosts:
|
|
hosts_with_params = [f"{host}/24,lzo" for host in distcc_hosts] + [
|
|
"localhost/`nproc`"
|
|
]
|
|
result.append('DISTCC_HOSTS="' + " ".join(hosts_with_params) + '"')
|
|
elif cache == "distcc":
|
|
result.append('DISTCC_HOSTS="localhost/`nproc`"')
|
|
|
|
if additional_pkgs:
|
|
# NOTE: This are the env for packages/build script
|
|
result.append("MAKE_APK=true")
|
|
result.append("MAKE_RPM=true")
|
|
result.append("MAKE_TGZ=true")
|
|
|
|
if with_binaries == "programs":
|
|
result.append("BINARY_OUTPUT=programs")
|
|
elif with_binaries == "tests":
|
|
result.append("ENABLE_TESTS=1")
|
|
result.append("BINARY_OUTPUT=tests")
|
|
cmake_flags.append("-DENABLE_TESTS=1")
|
|
|
|
if shared_libraries:
|
|
cmake_flags.append("-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1")
|
|
# We can't always build utils because it requires too much space, but
|
|
# we have to build them at least in some way in CI. The shared library
|
|
# build is probably the least heavy disk-wise.
|
|
cmake_flags.append("-DENABLE_UTILS=1")
|
|
# utils are not included into clickhouse-bundle, so build everything
|
|
build_target = "all"
|
|
|
|
if clang_tidy:
|
|
cmake_flags.append("-DENABLE_CLANG_TIDY=1")
|
|
cmake_flags.append("-DENABLE_TESTS=1")
|
|
cmake_flags.append("-DENABLE_EXAMPLES=1")
|
|
# Don't stop on first error to find more clang-tidy errors in one run.
|
|
result.append("NINJA_FLAGS=-k0")
|
|
|
|
cmake_flags.append("-DENABLE_UTILS=1")
|
|
# utils are not included into clickhouse-bundle, so build everything
|
|
build_target = "all"
|
|
|
|
if with_coverage:
|
|
cmake_flags.append("-DWITH_COVERAGE=1")
|
|
|
|
if version:
|
|
result.append(f"VERSION_STRING='{version}'")
|
|
|
|
if author:
|
|
result.append(f"AUTHOR='{author}'")
|
|
|
|
if official:
|
|
cmake_flags.append("-DCLICKHOUSE_OFFICIAL_BUILD=1")
|
|
|
|
result.append('CMAKE_FLAGS="' + " ".join(cmake_flags) + '"')
|
|
result.append(f"BUILD_TARGET={build_target}")
|
|
|
|
return result
|
|
|
|
|
|
def dir_name(name: str) -> str:
|
|
if not os.path.isabs(name):
|
|
name = os.path.abspath(os.path.join(os.getcwd(), name))
|
|
return name
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
|
|
parser = argparse.ArgumentParser(
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
description="ClickHouse building script using prebuilt Docker image",
|
|
)
|
|
parser.add_argument(
|
|
"--package-type",
|
|
choices=["deb", "binary", "coverity"],
|
|
required=True,
|
|
)
|
|
parser.add_argument(
|
|
"--clickhouse-repo-path",
|
|
default=os.path.join(os.path.dirname(SCRIPT_PATH), os.pardir, os.pardir),
|
|
type=dir_name,
|
|
help="ClickHouse git repository",
|
|
)
|
|
parser.add_argument("--output-dir", type=dir_name, required=True)
|
|
parser.add_argument("--build-type", choices=("debug", ""), default="")
|
|
|
|
parser.add_argument(
|
|
"--compiler",
|
|
choices=(
|
|
"clang-15",
|
|
"clang-15-darwin",
|
|
"clang-15-darwin-aarch64",
|
|
"clang-15-aarch64",
|
|
"clang-15-aarch64-v80compat",
|
|
"clang-15-ppc64le",
|
|
"clang-15-amd64-compat",
|
|
"clang-15-freebsd",
|
|
"gcc-11",
|
|
),
|
|
default="clang-15",
|
|
help="a compiler to use",
|
|
)
|
|
parser.add_argument(
|
|
"--sanitizer",
|
|
choices=("address", "thread", "memory", "undefined", ""),
|
|
default="",
|
|
)
|
|
|
|
parser.add_argument("--shared-libraries", action="store_true")
|
|
parser.add_argument("--clang-tidy", action="store_true")
|
|
parser.add_argument("--cache", choices=("ccache", "distcc", ""), default="")
|
|
parser.add_argument(
|
|
"--ccache_dir",
|
|
default=os.getenv("HOME", "") + "/.ccache",
|
|
type=dir_name,
|
|
help="a directory with ccache",
|
|
)
|
|
parser.add_argument("--distcc-hosts", nargs="+")
|
|
parser.add_argument("--force-build-image", action="store_true")
|
|
parser.add_argument("--version")
|
|
parser.add_argument("--author", default="clickhouse", help="a package author")
|
|
parser.add_argument("--official", action="store_true")
|
|
parser.add_argument("--additional-pkgs", action="store_true")
|
|
parser.add_argument("--with-coverage", action="store_true")
|
|
parser.add_argument(
|
|
"--with-binaries", choices=("programs", "tests", ""), default=""
|
|
)
|
|
parser.add_argument(
|
|
"--docker-image-version", default="latest", help="docker image tag to use"
|
|
)
|
|
parser.add_argument(
|
|
"--as-root", action="store_true", help="if the container should run as root"
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
image_name = f"clickhouse/{IMAGE_TYPE}-builder"
|
|
|
|
ch_root = args.clickhouse_repo_path
|
|
|
|
if args.additional_pkgs and args.package_type != "deb":
|
|
raise Exception("Can build additional packages only in deb build")
|
|
|
|
if args.with_binaries != "" and args.package_type != "deb":
|
|
raise Exception("Can add additional binaries only in deb build")
|
|
|
|
if args.with_binaries != "" and args.package_type == "deb":
|
|
logging.info("Should place %s to output", args.with_binaries)
|
|
|
|
dockerfile = os.path.join(ch_root, "docker/packager", IMAGE_TYPE, "Dockerfile")
|
|
image_with_version = image_name + ":" + args.docker_image_version
|
|
if not check_image_exists_locally(image_name) or args.force_build_image:
|
|
if not pull_image(image_with_version) or args.force_build_image:
|
|
build_image(image_with_version, dockerfile)
|
|
env_prepared = parse_env_variables(
|
|
args.build_type,
|
|
args.compiler,
|
|
args.sanitizer,
|
|
args.package_type,
|
|
args.cache,
|
|
args.distcc_hosts,
|
|
args.shared_libraries,
|
|
args.clang_tidy,
|
|
args.version,
|
|
args.author,
|
|
args.official,
|
|
args.additional_pkgs,
|
|
args.with_coverage,
|
|
args.with_binaries,
|
|
)
|
|
|
|
pre_build(args.clickhouse_repo_path, env_prepared)
|
|
run_docker_image_with_env(
|
|
image_name,
|
|
args.as_root,
|
|
args.output_dir,
|
|
env_prepared,
|
|
ch_root,
|
|
args.ccache_dir,
|
|
args.docker_image_version,
|
|
)
|
|
logging.info("Output placed into %s", args.output_dir)
|