From f2182db765151690754442baaa8cc13047ad41b0 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 24 Aug 2020 13:05:56 +0300 Subject: [PATCH 1/2] Separate stateless unbundled image --- docker/images.json | 11 +++ docker/packager/unbundled/Dockerfile | 56 +++++++++++++ docker/packager/unbundled/build.sh | 28 +++++++ docker/test/stateless_unbundled/Dockerfile | 84 +++++++++++++++++++ ...lickhouse-statelest-test-runner.Dockerfile | 15 ++++ docker/test/stateless_unbundled/run.sh | 65 ++++++++++++++ 6 files changed, 259 insertions(+) create mode 100644 docker/packager/unbundled/Dockerfile create mode 100755 docker/packager/unbundled/build.sh create mode 100644 docker/test/stateless_unbundled/Dockerfile create mode 100644 docker/test/stateless_unbundled/clickhouse-statelest-test-runner.Dockerfile create mode 100755 docker/test/stateless_unbundled/run.sh diff --git a/docker/images.json b/docker/images.json index c2e903e182d..9ea9e5ce1d8 100644 --- a/docker/images.json +++ b/docker/images.json @@ -132,5 +132,16 @@ "name": "yandex/clickhouse-test-base", "dependent": [ ] + }, + "docker/packager/unbundled": { + "name": "yandex/clickhouse-unbundled-builder", + "dependent": [ + "docker/test/stateless_unbundled" + ] + } + "docker/test/stateless_unbundled": { + "name": "yandex/clickhouse-stateless-unbundled-test", + "dependent": [ + ] } } diff --git a/docker/packager/unbundled/Dockerfile b/docker/packager/unbundled/Dockerfile new file mode 100644 index 00000000000..b52c1d47dd2 --- /dev/null +++ b/docker/packager/unbundled/Dockerfile @@ -0,0 +1,56 @@ +# docker build -t yandex/clickhouse-unbundled-builder . +FROM yandex/clickhouse-deb-builder + +# Libraries from OS are only needed to test the "unbundled" build (that is not used in production). +RUN apt-get --allow-unauthenticated update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get --allow-unauthenticated install --yes --no-install-recommends \ + libicu-dev \ + libreadline-dev \ + gperf \ + perl \ + pkg-config \ + devscripts \ + libc++-dev \ + libc++abi-dev \ + libboost-program-options-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + libboost-thread-dev \ + libboost-iostreams-dev \ + libboost-regex-dev \ + zlib1g-dev \ + liblz4-dev \ + libdouble-conversion-dev \ + librdkafka-dev \ + libpoconetssl62 \ + libpoco-dev \ + libgoogle-perftools-dev \ + libzstd-dev \ + libltdl-dev \ + libre2-dev \ + libjemalloc-dev \ + libmsgpack-dev \ + libcurl4-openssl-dev \ + opencl-headers \ + ocl-icd-libopencl1 \ + intel-opencl-icd \ + unixodbc-dev \ + odbcinst \ + tzdata \ + gperf \ + alien \ + libcapnp-dev \ + cmake \ + gdb \ + pigz \ + moreutils \ + libcctz-dev \ + libldap2-dev \ + libsasl2-dev \ + heimdal-multidev \ + libhyperscan-dev + +COPY build.sh / + +CMD ["/bin/bash", "/build.sh"] \ No newline at end of file diff --git a/docker/packager/unbundled/build.sh b/docker/packager/unbundled/build.sh new file mode 100755 index 00000000000..8b26bbb19cb --- /dev/null +++ b/docker/packager/unbundled/build.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -x -e + +# Update tzdata to the latest version. It is embedded into clickhouse binary. +sudo apt-get update && sudo apt-get install tzdata + +ccache --show-stats ||: +ccache --zero-stats ||: +build/release --no-pbuilder $ALIEN_PKGS | ts '%Y-%m-%d %H:%M:%S' +mv /*.deb /output +mv *.changes /output +mv *.buildinfo /output +mv /*.rpm /output ||: # if exists +mv /*.tgz /output ||: # if exists + +if [ -n "$BINARY_OUTPUT" ] && { [ "$BINARY_OUTPUT" = "programs" ] || [ "$BINARY_OUTPUT" = "tests" ] ;} +then + echo Place $BINARY_OUTPUT to output + mkdir /output/binary ||: # if exists + mv /build/obj-*/programs/clickhouse* /output/binary + if [ "$BINARY_OUTPUT" = "tests" ] + then + mv /build/obj-*/src/unit_tests_dbms /output/binary + fi +fi +ccache --show-stats ||: +ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||: diff --git a/docker/test/stateless_unbundled/Dockerfile b/docker/test/stateless_unbundled/Dockerfile new file mode 100644 index 00000000000..7de29fede72 --- /dev/null +++ b/docker/test/stateless_unbundled/Dockerfile @@ -0,0 +1,84 @@ +# docker build -t yandex/clickhouse-stateless-unbundled-test . +FROM yandex/clickhouse-test-base + +ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz" + +RUN apt-get --allow-unauthenticated update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get --allow-unauthenticated install --yes --no-install-recommends \ + alien \ + brotli \ + cmake \ + devscripts \ + expect \ + gdb \ + gperf \ + gperf \ + heimdal-multidev \ + intel-opencl-icd \ + libboost-filesystem-dev \ + libboost-iostreams-dev \ + libboost-program-options-dev \ + libboost-regex-dev \ + libboost-system-dev \ + libboost-thread-dev \ + libc++-dev \ + libc++abi-dev \ + libcapnp-dev \ + libcctz-dev \ + libcurl4-openssl-dev \ + libdouble-conversion-dev \ + libgoogle-perftools-dev \ + libhyperscan-dev \ + libicu-dev \ + libjemalloc-dev \ + libldap2-dev \ + libltdl-dev \ + liblz4-dev \ + libmsgpack-dev \ + libpoco-dev \ + libpoconetssl62 \ + librdkafka-dev \ + libre2-dev \ + libreadline-dev \ + libsasl2-dev \ + libzstd-dev \ + lsof \ + moreutils \ + ncdu \ + netcat-openbsd \ + ocl-icd-libopencl1 \ + odbcinst \ + opencl-headers \ + openssl \ + perl \ + pigz \ + pkg-config \ + python \ + python-lxml \ + python-requests \ + python-termcolor \ + qemu-user-static \ + sudo \ + telnet \ + tree \ + tzdata \ + unixodbc \ + unixodbc-dev \ + wget \ + zlib1g-dev \ + zookeeper \ + zookeeperd + +RUN mkdir -p /tmp/clickhouse-odbc-tmp \ + && wget --quiet -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \ + && cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \ + && odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \ + && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \ + && rm -rf /tmp/clickhouse-odbc-tmp + +ENV TZ=Europe/Moscow +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +COPY run.sh / +CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/stateless_unbundled/clickhouse-statelest-test-runner.Dockerfile b/docker/test/stateless_unbundled/clickhouse-statelest-test-runner.Dockerfile new file mode 100644 index 00000000000..562141ba147 --- /dev/null +++ b/docker/test/stateless_unbundled/clickhouse-statelest-test-runner.Dockerfile @@ -0,0 +1,15 @@ +# Since right now we can't set volumes to the docker during build, we split building container in stages: +# 1. build base container +# 2. run base conatiner with mounted volumes +# 3. commit container as image +FROM ubuntu:18.10 as clickhouse-test-runner-base + +# A volume where directory with clickhouse packages to be mounted, +# for later installing. +VOLUME /packages + +CMD apt-get update ;\ + DEBIAN_FRONTEND=noninteractive \ + apt install -y /packages/clickhouse-common-static_*.deb \ + /packages/clickhouse-client_*.deb \ + /packages/clickhouse-test_*.deb diff --git a/docker/test/stateless_unbundled/run.sh b/docker/test/stateless_unbundled/run.sh new file mode 100755 index 00000000000..2ff15ca9c6a --- /dev/null +++ b/docker/test/stateless_unbundled/run.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +set -e -x + +dpkg -i package_folder/clickhouse-common-static_*.deb +dpkg -i package_folder/clickhouse-common-static-dbg_*.deb +dpkg -i package_folder/clickhouse-server_*.deb +dpkg -i package_folder/clickhouse-client_*.deb +dpkg -i package_folder/clickhouse-test_*.deb + +mkdir -p /etc/clickhouse-server/dict_examples +ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/dict_examples/ +ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/dict_examples/ +ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/dict_examples/ +ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/metric_log.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/custom_settings_prefixes.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/ +ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/ +ln -s /usr/share/clickhouse-test/config/access_management.xml /etc/clickhouse-server/users.d/ +ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/ +ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/ +ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/ +ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/disks.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/secure_ports.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/clusters.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/graphite.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/server.key /etc/clickhouse-server/ +ln -s /usr/share/clickhouse-test/config/server.crt /etc/clickhouse-server/ +ln -s /usr/share/clickhouse-test/config/dhparam.pem /etc/clickhouse-server/ + +# Retain any pre-existing config and allow ClickHouse to load it if required +ln -s --backup=simple --suffix=_original.xml \ + /usr/share/clickhouse-test/config/query_masking_rules.xml /etc/clickhouse-server/config.d/ + +if [[ -n "$USE_POLYMORPHIC_PARTS" ]] && [[ "$USE_POLYMORPHIC_PARTS" -eq 1 ]]; then + ln -s /usr/share/clickhouse-test/config/polymorphic_parts.xml /etc/clickhouse-server/config.d/ +fi +if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then + ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/ + ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/ +fi + +ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml + +echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment +echo "TSAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment +echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment +echo "ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment +echo "UBSAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment +echo "LLVM_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment + +service zookeeper start +sleep 5 +service clickhouse-server start && sleep 5 + +if cat /usr/bin/clickhouse-test | grep -q -- "--use-skip-list"; then + SKIP_LIST_OPT="--use-skip-list" +fi + +clickhouse-test --testname --shard --zookeeper "$SKIP_LIST_OPT" $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt From 7a98a0fdb02bc31b100abc5daa79fe0743b0296b Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Mon, 24 Aug 2020 13:27:09 +0300 Subject: [PATCH 2/2] Update images.json --- docker/images.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/images.json b/docker/images.json index 9ea9e5ce1d8..723d5fc3e7d 100644 --- a/docker/images.json +++ b/docker/images.json @@ -138,7 +138,7 @@ "dependent": [ "docker/test/stateless_unbundled" ] - } + }, "docker/test/stateless_unbundled": { "name": "yandex/clickhouse-stateless-unbundled-test", "dependent": [