diff --git a/docker/images.json b/docker/images.json index 0ab1688efb0..e9e83f7859e 100644 --- a/docker/images.json +++ b/docker/images.json @@ -2,6 +2,7 @@ "docker/packager/deb": { "name": "yandex/clickhouse-deb-builder", "dependent": [ + "docker/packager/unbundled", "docker/test/stateless", "docker/test/stateless_with_coverage", "docker/test/stateless_pytest", @@ -15,6 +16,10 @@ "docker/test/pvs" ] }, + "docker/packager/unbundled": { + "name": "yandex/clickhouse-unbundled-builder", + "dependent": [] + }, "docker/test/coverage": { "name": "yandex/clickhouse-coverage", "dependent": [] diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index c1260b5c7ff..b1f711509fb 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -1,9 +1,9 @@ # docker build -t yandex/clickhouse-deb-builder . -FROM ubuntu:19.10 +FROM ubuntu:20.04 RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnupg RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - -RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list +RUN echo "deb [trusted=yes] http://apt.llvm.org/focal/ llvm-toolchain-focal-11 main" >> /etc/apt/sources.list # initial packages RUN apt-get --allow-unauthenticated update -y \ @@ -25,13 +25,17 @@ RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb RUN chmod +x dpkg-deb RUN cp dpkg-deb /usr/bin - -# Libraries from OS are only needed to test the "unbundled" build (that is not used in production). RUN apt-get --allow-unauthenticated update -y \ && env DEBIAN_FRONTEND=noninteractive \ apt-get --allow-unauthenticated install --yes --no-install-recommends \ + gcc-10 \ + g++-10 \ gcc-9 \ g++-9 \ + llvm-11 \ + clang-11 \ + lld-11 \ + clang-tidy-11 \ llvm-10 \ clang-10 \ lld-10 \ @@ -39,54 +43,19 @@ RUN apt-get --allow-unauthenticated update -y \ clang-9 \ lld-9 \ clang-tidy-9 \ - libicu-dev \ - libreadline-dev \ - gperf \ ninja-build \ perl \ pkg-config \ devscripts \ debhelper \ git \ - libc++-dev \ - libc++abi-dev \ - libboost-program-options-dev \ - libboost-system-dev \ - libboost-filesystem-dev \ - libboost-thread-dev \ - libboost-iostreams-dev \ - libboost-regex-dev \ - zlib1g-dev \ - liblz4-dev \ - libdouble-conversion-dev \ - librdkafka-dev \ - libpoconetssl62 \ - libpoco-dev \ - libgoogle-perftools-dev \ - libzstd-dev \ - libltdl-dev \ - libre2-dev \ - libjemalloc-dev \ - libmsgpack-dev \ - libcurl4-openssl-dev \ - opencl-headers \ - ocl-icd-libopencl1 \ - intel-opencl-icd \ - unixodbc-dev \ - odbcinst \ tzdata \ gperf \ alien \ - libcapnp-dev \ cmake \ gdb \ - pigz \ moreutils \ - libcctz-dev \ - libldap2-dev \ - libsasl2-dev \ - heimdal-multidev \ - libhyperscan-dev + pigz # This symlink required by gcc to find lld compiler diff --git a/docker/packager/packager b/docker/packager/packager index bc97429336c..63652da2439 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -11,6 +11,7 @@ SCRIPT_PATH = os.path.realpath(__file__) IMAGE_MAP = { "deb": "yandex/clickhouse-deb-builder", "binary": "yandex/clickhouse-binary-builder", + "unbundled": "yandex/clickhouse-unbundled-builder" } def check_image_exists_locally(image_name): @@ -176,7 +177,9 @@ if __name__ == "__main__": parser.add_argument("--clickhouse-repo-path", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir)) parser.add_argument("--output-dir", required=True) parser.add_argument("--build-type", choices=("debug", ""), default="") - parser.add_argument("--compiler", choices=("clang-10-darwin", "clang-10-aarch64", "clang-10-freebsd", "gcc-9", "clang-10"), default="gcc-9") + parser.add_argument("--compiler", choices=("clang-10", "clang-10-darwin", "clang-10-aarch64", "clang-10-freebsd", + "clang-11", "clang-11-darwin", "clang-11-aarch64", "clang-11-freebsd", + "gcc-9", "gcc-10"), default="gcc-9") parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="") parser.add_argument("--unbundled", action="store_true") parser.add_argument("--split-binary", action="store_true") @@ -197,7 +200,7 @@ if __name__ == "__main__": if not os.path.isabs(args.output_dir): args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir)) - image_type = 'binary' if args.package_type == 'performance' else args.package_type + image_type = 'binary' if args.package_type == 'performance' else 'unbundled' if args.unbundled else args.package_type image_name = IMAGE_MAP[image_type] if not os.path.isabs(args.clickhouse_repo_path): diff --git a/docker/packager/unbundled/Dockerfile b/docker/packager/unbundled/Dockerfile new file mode 100644 index 00000000000..d099be6cb90 --- /dev/null +++ b/docker/packager/unbundled/Dockerfile @@ -0,0 +1,56 @@ +# docker build -t yandex/clickhouse-unbundled-builder . +FROM yandex/clickhouse-deb-builder + +# Libraries from OS are only needed to test the "unbundled" build (that is not used in production). +RUN apt-get --allow-unauthenticated update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get --allow-unauthenticated install --yes --no-install-recommends \ + libicu-dev \ + libreadline-dev \ + gperf \ + perl \ + pkg-config \ + devscripts \ + libc++-dev \ + libc++abi-dev \ + libboost-program-options-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + libboost-thread-dev \ + libboost-iostreams-dev \ + libboost-regex-dev \ + zlib1g-dev \ + liblz4-dev \ + libdouble-conversion-dev \ + librdkafka-dev \ + libpoconetssl62 \ + libpoco-dev \ + libgoogle-perftools-dev \ + libzstd-dev \ + libltdl-dev \ + libre2-dev \ + libjemalloc-dev \ + libmsgpack-dev \ + libcurl4-openssl-dev \ + opencl-headers \ + ocl-icd-libopencl1 \ + intel-opencl-icd \ + unixodbc-dev \ + odbcinst \ + tzdata \ + gperf \ + alien \ + libcapnp-dev \ + cmake \ + gdb \ + pigz \ + moreutils \ + libcctz-dev \ + libldap2-dev \ + libsasl2-dev \ + heimdal-multidev \ + libhyperscan-dev + +COPY build.sh / + +CMD ["/bin/bash", "/build.sh"] diff --git a/docker/packager/unbundled/build.sh b/docker/packager/unbundled/build.sh new file mode 100755 index 00000000000..9fd246bcc33 --- /dev/null +++ b/docker/packager/unbundled/build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -x -e + +# Update tzdata to the latest version. It is embedded into clickhouse binary. +sudo apt-get update && sudo apt-get install tzdata + +ccache --show-stats ||: +ccache --zero-stats ||: +build/release --no-pbuilder $ALIEN_PKGS | ts '%Y-%m-%d %H:%M:%S' +mv /*.deb /output +mv *.changes /output +mv *.buildinfo /output +mv /*.rpm /output ||: # if exists +mv /*.tgz /output ||: # if exists + +ccache --show-stats ||: +ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||: