CLICKHOUSE-3968: Add script for package creation in docker

docker/packager/README.md

@@ -0,0 +1,41 @@
+Allow to build ClickHouse in Docker for different platforms with different
+compilers and build settings. Correctly configured Docker daemon is single dependency.
+
+Usage:
+
+Build deb package with gcc-8 in debug mode:
+```
+$ mkdir deb/test_output
+$ packager/packager --output-dir deb/test_output/ --package-type deb --compiler=gcc-8 --build-type=debug --clickhouse-repo-path ../
+$ ls -l deb/test_output
+-rw-r--r-- 1 root root      3730 clickhouse-client_18.14.2+debug_all.deb
+-rw-r--r-- 1 root root  84221888 clickhouse-common-static_18.14.2+debug_amd64.deb
+-rw-r--r-- 1 root root 255967314 clickhouse-common-static-dbg_18.14.2+debug_amd64.deb
+-rw-r--r-- 1 root root     14940 clickhouse-server_18.14.2+debug_all.deb
+-rw-r--r-- 1 root root 340206010 clickhouse-server-base_18.14.2+debug_amd64.deb
+-rw-r--r-- 1 root root      7900 clickhouse-server-common_18.14.2+debug_all.deb
+-rw-r--r-- 1 root root   2880432 clickhouse-test_18.14.2+debug_all.deb
+
+```
+
+Build ClickHouse binary with clang-6.0 and address sanitizer in relwithdebuginfo
+mode:
+```
+$ mkdir $HOME/some_clickhouse
+$ ./packager --output-dir=$HOME/some_clickhouse --package-type binary --compiler=clang-6.0 --sanitizer=address
+$ ls -l $HOME/some_clickhouse
+-rwxr-xr-x 1 root root 787061952  clickhouse
+lrwxrwxrwx 1 root root        10  clickhouse-benchmark -> clickhouse
+lrwxrwxrwx 1 root root        10  clickhouse-clang -> clickhouse
+lrwxrwxrwx 1 root root        10  clickhouse-client -> clickhouse
+lrwxrwxrwx 1 root root        10  clickhouse-compressor -> clickhouse
+lrwxrwxrwx 1 root root        10  clickhouse-copier -> clickhouse
+lrwxrwxrwx 1 root root        10  clickhouse-extract-from-config -> clickhouse
+lrwxrwxrwx 1 root root        10  clickhouse-format -> clickhouse
+lrwxrwxrwx 1 root root        10  clickhouse-lld -> clickhouse
+lrwxrwxrwx 1 root root        10  clickhouse-local -> clickhouse
+lrwxrwxrwx 1 root root        10  clickhouse-obfuscator -> clickhouse
+lrwxrwxrwx 1 root root        10  clickhouse-odbc-bridge -> clickhouse
+lrwxrwxrwx 1 root root        10  clickhouse-performance-test -> clickhouse
+lrwxrwxrwx 1 root root        10  clickhouse-server -> clickhouse
+```

docker/packager/binary/Dockerfile

@@ -0,0 +1,23 @@
+FROM ubuntu:18.04
+
+RUN apt-get update -y \
+    && env DEBIAN_FRONTEND=noninteractive \
+        apt-get install --yes --no-install-recommends \
+            bash \
+            cmake \
+            curl \
+            gcc-7 \
+            g++-7 \
+            gcc-8 \
+            g++-8 \
+            clang-6.0 \
+            clang++-6.0 \
+            lld-6.0 \
+            libclang-6.0-dev \
+            libicu-dev \
+            liblld-6.0-dev \
+            libreadline-dev \
+            ninja-build \
+            git
+
+CMD mkdir -p build/build_result && cd build/build_result && cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER && ninja && mv ./dbms/programs/clickhouse* /output

docker/packager/deb/Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:18.04
+
+RUN apt-get update -y \
+    && env DEBIAN_FRONTEND=noninteractive \
+        apt-get install --yes --no-install-recommends \
+            bash \
+            fakeroot \
+            cmake \
+            curl \
+            gcc-7 \
+            g++-7 \
+            gcc-8 \
+            g++-8 \
+            clang-6.0 \
+            clang++-6.0 \
+            lld-6.0 \
+            libclang-6.0-dev \
+            libicu-dev \
+            liblld-6.0-dev \
+            libreadline-dev \
+            ninja-build \
+            perl \
+            pkg-config \
+            devscripts \
+            debhelper \
+            git
+
+CMD /bin/bash build/release --no-pbuilder && mv /*.deb /output

docker/packager/packager

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+import subprocess
+import os
+import argparse
+import logging
+
+IMAGE_MAP = {
+    "deb": "yandex/clickhouse-deb-builder",
+    "binary": "yandex/clickhouse-binary-builder",
+}
+
+
+def check_image_exists_locally(image_name):
+    try:
+        output = subprocess.check_output("docker images -q {} 2> /dev/null".format(image_name), shell=True)
+        return output != ""
+    except subprocess.CalledProcessError as ex:
+        return False
+
+def pull_image(image_name):
+    try:
+        subprocess.check_call("docker pull {}".format(image_name), shell=True)
+        return True
+    except subprocess.CalledProcessError as ex:
+        logging.info("Cannot pull image {}".format(image_name))
+        return False
+
+def build_image(image_name, filepath):
+    subprocess.check_call("docker build --network=host -t {} -f {} .".format(image_name, filepath), shell=True)
+
+def run_image_with_env(image_name, output, env_variables, ch_root):
+    env_part = " -e ".join(env_variables)
+    if env_part:
+        env_part = " -e " + env_part
+
+    cmd = "docker run --network=host --rm --volume={output_path}:/output --volume={ch_root}:/build {env} -it {img_name}".format(
+        output_path=output,
+        ch_root=ch_root,
+        env=env_part,
+        img_name=image_name
+    )
+
+    logging.info("Will build ClickHouse pkg with cmd: '{}'".format(cmd))
+
+    subprocess.check_call(cmd, shell=True)
+
+def parse_env_variables(build_type, compiler, sanitizer, package_type):
+    result = []
+    if package_type == "deb":
+        result.append("DEB_CC={}".format(compiler))
+        result.append("DEB_CXX={}".format(compiler.replace('gcc', 'g++').replace('clang', 'clang++')))
+    elif package_type == "binary":
+        result.append("CC={}".format(compiler))
+        result.append("CXX={}".format(compiler.replace('gcc', 'g++').replace('clang', 'clang++')))
+
+    if sanitizer:
+        result.append("SANITIZER={}".format(sanitizer))
+    if build_type:
+        result.append("BUILD_TYPE={}".format(build_type))
+
+
+    return result
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
+    parser = argparse.ArgumentParser(description="ClickHouse building script via docker")
+    parser.add_argument("--package-type", choices=IMAGE_MAP.keys())
+    parser.add_argument("--clickhouse-repo-path", default="../../")
+    parser.add_argument("--output-dir", required=True)
+    parser.add_argument("--build-type", choices=("debug", ""), default="")
+    parser.add_argument("--compiler", choices=("clang-6.0", "gcc-7", "gcc-8"), default="gcc-7")
+    parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="")
+    parser.add_argument("--force-build-image", action="store_true")
+
+    args = parser.parse_args()
+    if not os.path.isabs(args.output_dir):
+        args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir))
+
+    image_name = IMAGE_MAP[args.package_type]
+
+    if not os.path.isabs(args.clickhouse_repo_path):
+        ch_root = os.path.abspath(os.path.join(os.getcwd(), args.clickhouse_repo_path))
+    else:
+        ch_root = args.clickhouse_repo_path
+
+    dockerfile = os.path.join(ch_root, "docker/packager", args.package_type, "Dockerfile")
+    try:
+        if not check_image_exists_locally(image_name) or args.force_build_image:
+            if not pull_image(image_name) or args.force_build_image:
+                build_image(image_name, dockerfile)
+        run_image_with_env(image_name, args.output_dir, parse_env_variables(args.build_type, args.compiler, args.sanitizer, args.package_type), ch_root)
+        logging.info("Output placed into {}".format(args.output_dir))
+    except Exception as ex:
+        logging.info("Exception occured {}".format(ex))