Add separate docker container for unbundled build

docker/images.json

@@ -2,6 +2,7 @@
     "docker/packager/deb": {
         "name": "yandex/clickhouse-deb-builder",
         "dependent": [
+            "docker/packager/unbundled",
             "docker/test/stateless",
             "docker/test/stateless_with_coverage",
             "docker/test/stateless_pytest",
@@ -15,6 +16,10 @@
             "docker/test/pvs"
         ]
     },
+    "docker/packager/unbundled": {
+        "name": "yandex/clickhouse-unbundled-builder",
+        "dependent": []
+    },
     "docker/test/coverage": {
         "name": "yandex/clickhouse-coverage",
         "dependent": []

docker/packager/deb/Dockerfile

@@ -1,9 +1,9 @@
 # docker build -t yandex/clickhouse-deb-builder .
-FROM ubuntu:19.10
+FROM ubuntu:20.04
 
 RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnupg
 RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
-RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list
+RUN echo "deb [trusted=yes] http://apt.llvm.org/focal/ llvm-toolchain-focal-11 main" >> /etc/apt/sources.list
 
 # initial packages
 RUN apt-get --allow-unauthenticated update -y \
@@ -25,13 +25,17 @@ RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb
 RUN chmod +x dpkg-deb
 RUN cp dpkg-deb /usr/bin
 
-
-# Libraries from OS are only needed to test the "unbundled" build (that is not used in production).
 RUN apt-get --allow-unauthenticated update -y \
     && env DEBIAN_FRONTEND=noninteractive \
         apt-get --allow-unauthenticated install --yes --no-install-recommends \
+            gcc-10 \
+            g++-10 \
             gcc-9 \
             g++-9 \
+            llvm-11 \
+            clang-11 \
+            lld-11 \
+            clang-tidy-11 \
             llvm-10 \
             clang-10 \
             lld-10 \
@@ -39,54 +43,18 @@ RUN apt-get --allow-unauthenticated update -y \
             clang-9 \
             lld-9 \
             clang-tidy-9 \
-            libicu-dev \
-            libreadline-dev \
-            gperf \
             ninja-build \
             perl \
             pkg-config \
             devscripts \
             debhelper \
             git \
-            libc++-dev \
-            libc++abi-dev \
-            libboost-program-options-dev \
-            libboost-system-dev \
-            libboost-filesystem-dev \
-            libboost-thread-dev \
-            libboost-iostreams-dev \
-            libboost-regex-dev \
-            zlib1g-dev \
-            liblz4-dev \
-            libdouble-conversion-dev \
-            librdkafka-dev \
-            libpoconetssl62 \
-            libpoco-dev \
-            libgoogle-perftools-dev \
-            libzstd-dev \
-            libltdl-dev \
-            libre2-dev \
-            libjemalloc-dev \
-            libmsgpack-dev \
-            libcurl4-openssl-dev \
-            opencl-headers \
-            ocl-icd-libopencl1 \
-            intel-opencl-icd \
-            unixodbc-dev \
-            odbcinst \
             tzdata \
             gperf \
             alien \
-            libcapnp-dev \
             cmake \
             gdb  \
-            pigz \
+            moreutils
-            moreutils \
-            libcctz-dev \
-            libldap2-dev \
-            libsasl2-dev \
-            heimdal-multidev \
-            libhyperscan-dev
 
 
 # This symlink required by gcc to find lld compiler

docker/packager/packager

@@ -11,6 +11,7 @@ SCRIPT_PATH = os.path.realpath(__file__)
 IMAGE_MAP = {
     "deb": "yandex/clickhouse-deb-builder",
     "binary": "yandex/clickhouse-binary-builder",
+    "unbundled": "yandex/clickhouse-unbundled-builder"
 }
 
 def check_image_exists_locally(image_name):
@@ -176,7 +177,9 @@ if __name__ == "__main__":
     parser.add_argument("--clickhouse-repo-path", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir))
     parser.add_argument("--output-dir", required=True)
     parser.add_argument("--build-type", choices=("debug", ""), default="")
-    parser.add_argument("--compiler", choices=("clang-10-darwin", "clang-10-aarch64", "clang-10-freebsd", "gcc-9", "clang-10"), default="gcc-9")
+    parser.add_argument("--compiler", choices=("clang-10", "clang-10-darwin", "clang-10-aarch64", "clang-10-freebsd",
+                                               "clang-11", "clang-11-darwin", "clang-11-aarch64", "clang-11-freebsd",
+                                               "gcc-9", "gcc-10"), default="gcc-9")
     parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="")
     parser.add_argument("--unbundled", action="store_true")
     parser.add_argument("--split-binary", action="store_true")
@@ -197,7 +200,7 @@ if __name__ == "__main__":
     if not os.path.isabs(args.output_dir):
         args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir))
 
-    image_type = 'binary' if args.package_type == 'performance' else args.package_type
+    image_type = 'binary' if args.package_type == 'performance' else 'unbundled' if args.unbundled else args.package_type
     image_name = IMAGE_MAP[image_type]
 
     if not os.path.isabs(args.clickhouse_repo_path):

docker/packager/unbundled/Dockerfile

@@ -0,0 +1,56 @@
+# docker build -t yandex/clickhouse-unbundled-builder .
+FROM yandex/clickhouse-deb-builder
+
+# Libraries from OS are only needed to test the "unbundled" build (that is not used in production).
+RUN apt-get --allow-unauthenticated update -y \
+    && env DEBIAN_FRONTEND=noninteractive \
+        apt-get --allow-unauthenticated install --yes --no-install-recommends \
+            libicu-dev \
+            libreadline-dev \
+            gperf \
+            perl \
+            pkg-config \
+            devscripts \
+            libc++-dev \
+            libc++abi-dev \
+            libboost-program-options-dev \
+            libboost-system-dev \
+            libboost-filesystem-dev \
+            libboost-thread-dev \
+            libboost-iostreams-dev \
+            libboost-regex-dev \
+            zlib1g-dev \
+            liblz4-dev \
+            libdouble-conversion-dev \
+            librdkafka-dev \
+            libpoconetssl62 \
+            libpoco-dev \
+            libgoogle-perftools-dev \
+            libzstd-dev \
+            libltdl-dev \
+            libre2-dev \
+            libjemalloc-dev \
+            libmsgpack-dev \
+            libcurl4-openssl-dev \
+            opencl-headers \
+            ocl-icd-libopencl1 \
+            intel-opencl-icd \
+            unixodbc-dev \
+            odbcinst \
+            tzdata \
+            gperf \
+            alien \
+            libcapnp-dev \
+            cmake \
+            gdb  \
+            pigz \
+            moreutils \
+            libcctz-dev \
+            libldap2-dev \
+            libsasl2-dev \
+            heimdal-multidev \
+            libhyperscan-dev
+
+COPY build.sh /
+
+CMD ["/bin/bash", "/build.sh"]

docker/packager/unbundled/build.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -x -e
+
+# Update tzdata to the latest version. It is embedded into clickhouse binary.
+sudo apt-get update && sudo apt-get install tzdata
+
+ccache --show-stats ||:
+ccache --zero-stats ||:
+build/release --no-pbuilder $ALIEN_PKGS | ts '%Y-%m-%d %H:%M:%S'
+mv /*.deb /output
+mv *.changes /output
+mv *.buildinfo /output
+mv /*.rpm /output ||: # if exists
+mv /*.tgz /output ||: # if exists
+
+if [ -n "$BINARY_OUTPUT" ] && { [ "$BINARY_OUTPUT" = "programs" ] || [ "$BINARY_OUTPUT" = "tests" ] ;}
+then
+  echo Place $BINARY_OUTPUT to output
+  mkdir /output/binary ||: # if exists
+  mv /build/obj-*/programs/clickhouse* /output/binary
+  if [ "$BINARY_OUTPUT" = "tests" ]
+  then
+    mv /build/obj-*/src/unit_tests_dbms /output/binary
+  fi
+fi
+ccache --show-stats ||:
+ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||: