diff --git a/docker/packager/README.md b/docker/packager/README.md new file mode 100644 index 00000000000..293f57902a5 --- /dev/null +++ b/docker/packager/README.md @@ -0,0 +1,41 @@ +Allow to build ClickHouse in Docker for different platforms with different +compilers and build settings. Correctly configured Docker daemon is single dependency. + +Usage: + +Build deb package with gcc-8 in debug mode: +``` +$ mkdir deb/test_output +$ packager/packager --output-dir deb/test_output/ --package-type deb --compiler=gcc-8 --build-type=debug --clickhouse-repo-path ../ +$ ls -l deb/test_output +-rw-r--r-- 1 root root 3730 clickhouse-client_18.14.2+debug_all.deb +-rw-r--r-- 1 root root 84221888 clickhouse-common-static_18.14.2+debug_amd64.deb +-rw-r--r-- 1 root root 255967314 clickhouse-common-static-dbg_18.14.2+debug_amd64.deb +-rw-r--r-- 1 root root 14940 clickhouse-server_18.14.2+debug_all.deb +-rw-r--r-- 1 root root 340206010 clickhouse-server-base_18.14.2+debug_amd64.deb +-rw-r--r-- 1 root root 7900 clickhouse-server-common_18.14.2+debug_all.deb +-rw-r--r-- 1 root root 2880432 clickhouse-test_18.14.2+debug_all.deb + +``` + +Build ClickHouse binary with clang-6.0 and address sanitizer in relwithdebuginfo +mode: +``` +$ mkdir $HOME/some_clickhouse +$ ./packager --output-dir=$HOME/some_clickhouse --package-type binary --compiler=clang-6.0 --sanitizer=address +$ ls -l $HOME/some_clickhouse +-rwxr-xr-x 1 root root 787061952 clickhouse +lrwxrwxrwx 1 root root 10 clickhouse-benchmark -> clickhouse +lrwxrwxrwx 1 root root 10 clickhouse-clang -> clickhouse +lrwxrwxrwx 1 root root 10 clickhouse-client -> clickhouse +lrwxrwxrwx 1 root root 10 clickhouse-compressor -> clickhouse +lrwxrwxrwx 1 root root 10 clickhouse-copier -> clickhouse +lrwxrwxrwx 1 root root 10 clickhouse-extract-from-config -> clickhouse +lrwxrwxrwx 1 root root 10 clickhouse-format -> clickhouse +lrwxrwxrwx 1 root root 10 clickhouse-lld -> clickhouse +lrwxrwxrwx 1 root root 10 clickhouse-local -> clickhouse +lrwxrwxrwx 1 root root 10 clickhouse-obfuscator -> clickhouse +lrwxrwxrwx 1 root root 10 clickhouse-odbc-bridge -> clickhouse +lrwxrwxrwx 1 root root 10 clickhouse-performance-test -> clickhouse +lrwxrwxrwx 1 root root 10 clickhouse-server -> clickhouse +``` diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile new file mode 100644 index 00000000000..49b757e1b00 --- /dev/null +++ b/docker/packager/binary/Dockerfile @@ -0,0 +1,23 @@ +FROM ubuntu:18.04 + +RUN apt-get update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends \ + bash \ + cmake \ + curl \ + gcc-7 \ + g++-7 \ + gcc-8 \ + g++-8 \ + clang-6.0 \ + clang++-6.0 \ + lld-6.0 \ + libclang-6.0-dev \ + libicu-dev \ + liblld-6.0-dev \ + libreadline-dev \ + ninja-build \ + git + +CMD mkdir -p build/build_result && cd build/build_result && cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER && ninja && mv ./dbms/programs/clickhouse* /output diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile new file mode 100644 index 00000000000..9e0abd3afd4 --- /dev/null +++ b/docker/packager/deb/Dockerfile @@ -0,0 +1,28 @@ +FROM ubuntu:18.04 + +RUN apt-get update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends \ + bash \ + fakeroot \ + cmake \ + curl \ + gcc-7 \ + g++-7 \ + gcc-8 \ + g++-8 \ + clang-6.0 \ + clang++-6.0 \ + lld-6.0 \ + libclang-6.0-dev \ + libicu-dev \ + liblld-6.0-dev \ + libreadline-dev \ + ninja-build \ + perl \ + pkg-config \ + devscripts \ + debhelper \ + git + +CMD /bin/bash build/release --no-pbuilder && mv /*.deb /output diff --git a/docker/packager/packager b/docker/packager/packager new file mode 100755 index 00000000000..50e6cb22363 --- /dev/null +++ b/docker/packager/packager @@ -0,0 +1,95 @@ +#!/usr/bin/env python +#-*- coding: utf-8 -*- +import subprocess +import os +import argparse +import logging + +IMAGE_MAP = { + "deb": "yandex/clickhouse-deb-builder", + "binary": "yandex/clickhouse-binary-builder", +} + + +def check_image_exists_locally(image_name): + try: + output = subprocess.check_output("docker images -q {} 2> /dev/null".format(image_name), shell=True) + return output != "" + except subprocess.CalledProcessError as ex: + return False + +def pull_image(image_name): + try: + subprocess.check_call("docker pull {}".format(image_name), shell=True) + return True + except subprocess.CalledProcessError as ex: + logging.info("Cannot pull image {}".format(image_name)) + return False + +def build_image(image_name, filepath): + subprocess.check_call("docker build --network=host -t {} -f {} .".format(image_name, filepath), shell=True) + +def run_image_with_env(image_name, output, env_variables, ch_root): + env_part = " -e ".join(env_variables) + if env_part: + env_part = " -e " + env_part + + cmd = "docker run --network=host --rm --volume={output_path}:/output --volume={ch_root}:/build {env} -it {img_name}".format( + output_path=output, + ch_root=ch_root, + env=env_part, + img_name=image_name + ) + + logging.info("Will build ClickHouse pkg with cmd: '{}'".format(cmd)) + + subprocess.check_call(cmd, shell=True) + +def parse_env_variables(build_type, compiler, sanitizer, package_type): + result = [] + if package_type == "deb": + result.append("DEB_CC={}".format(compiler)) + result.append("DEB_CXX={}".format(compiler.replace('gcc', 'g++').replace('clang', 'clang++'))) + elif package_type == "binary": + result.append("CC={}".format(compiler)) + result.append("CXX={}".format(compiler.replace('gcc', 'g++').replace('clang', 'clang++'))) + + if sanitizer: + result.append("SANITIZER={}".format(sanitizer)) + if build_type: + result.append("BUILD_TYPE={}".format(build_type)) + + + return result + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') + parser = argparse.ArgumentParser(description="ClickHouse building script via docker") + parser.add_argument("--package-type", choices=IMAGE_MAP.keys()) + parser.add_argument("--clickhouse-repo-path", default="../../") + parser.add_argument("--output-dir", required=True) + parser.add_argument("--build-type", choices=("debug", ""), default="") + parser.add_argument("--compiler", choices=("clang-6.0", "gcc-7", "gcc-8"), default="gcc-7") + parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="") + parser.add_argument("--force-build-image", action="store_true") + + args = parser.parse_args() + if not os.path.isabs(args.output_dir): + args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir)) + + image_name = IMAGE_MAP[args.package_type] + + if not os.path.isabs(args.clickhouse_repo_path): + ch_root = os.path.abspath(os.path.join(os.getcwd(), args.clickhouse_repo_path)) + else: + ch_root = args.clickhouse_repo_path + + dockerfile = os.path.join(ch_root, "docker/packager", args.package_type, "Dockerfile") + try: + if not check_image_exists_locally(image_name) or args.force_build_image: + if not pull_image(image_name) or args.force_build_image: + build_image(image_name, dockerfile) + run_image_with_env(image_name, args.output_dir, parse_env_variables(args.build_type, args.compiler, args.sanitizer, args.package_type), ch_root) + logging.info("Output placed into {}".format(args.output_dir)) + except Exception as ex: + logging.info("Exception occured {}".format(ex))