Merge pull request #13995 from qoega/stateless-unbundled

Separate stateless unbundled image
2024-09-20 00:30:49 +00:00 · 2020-08-24 19:49:27 +03:00 · 2020-08-24 19:49:27 +03:00 · 20bff27eef
commit 20bff27eef
parent d806e0c052 7a98a0fdb0
6 changed files with 259 additions and 0 deletions
--- a/docker/images.json
+++ b/docker/images.json
@ -132,5 +132,16 @@
         "name": "yandex/clickhouse-test-base",
         "dependent": [
         ]
+    },
+    "docker/packager/unbundled": {
+         "name": "yandex/clickhouse-unbundled-builder",
+         "dependent": [
+            "docker/test/stateless_unbundled"
+         ]
+    },
+    "docker/test/stateless_unbundled": {
+         "name": "yandex/clickhouse-stateless-unbundled-test",
+         "dependent": [
+         ]
    }
 }
--- a/docker/packager/unbundled/Dockerfile
+++ b/docker/packager/unbundled/Dockerfile
@ -0,0 +1,56 @@
+# docker build -t yandex/clickhouse-unbundled-builder .
+FROM yandex/clickhouse-deb-builder
+
+# Libraries from OS are only needed to test the "unbundled" build (that is not used in production).
+RUN apt-get --allow-unauthenticated update -y \
+    && env DEBIAN_FRONTEND=noninteractive \
+        apt-get --allow-unauthenticated install --yes --no-install-recommends \
+            libicu-dev \
+            libreadline-dev \
+            gperf \
+            perl \
+            pkg-config \
+            devscripts \
+            libc++-dev \
+            libc++abi-dev \
+            libboost-program-options-dev \
+            libboost-system-dev \
+            libboost-filesystem-dev \
+            libboost-thread-dev \
+            libboost-iostreams-dev \
+            libboost-regex-dev \
+            zlib1g-dev \
+            liblz4-dev \
+            libdouble-conversion-dev \
+            librdkafka-dev \
+            libpoconetssl62 \
+            libpoco-dev \
+            libgoogle-perftools-dev \
+            libzstd-dev \
+            libltdl-dev \
+            libre2-dev \
+            libjemalloc-dev \
+            libmsgpack-dev \
+            libcurl4-openssl-dev \
+            opencl-headers \
+            ocl-icd-libopencl1 \
+            intel-opencl-icd \
+            unixodbc-dev \
+            odbcinst \
+            tzdata \
+            gperf \
+            alien \
+            libcapnp-dev \
+            cmake \
+            gdb  \
+            pigz \
+            moreutils \
+            libcctz-dev \
+            libldap2-dev \
+            libsasl2-dev \
+            heimdal-multidev \
+            libhyperscan-dev
+
+COPY build.sh /
+
+CMD ["/bin/bash", "/build.sh"]
--- a/docker/packager/unbundled/build.sh
+++ b/docker/packager/unbundled/build.sh
@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -x -e
+
+# Update tzdata to the latest version. It is embedded into clickhouse binary.
+sudo apt-get update && sudo apt-get install tzdata
+
+ccache --show-stats ||:
+ccache --zero-stats ||:
+build/release --no-pbuilder $ALIEN_PKGS | ts '%Y-%m-%d %H:%M:%S'
+mv /*.deb /output
+mv *.changes /output
+mv *.buildinfo /output
+mv /*.rpm /output ||: # if exists
+mv /*.tgz /output ||: # if exists
+
+if [ -n "$BINARY_OUTPUT" ] && { [ "$BINARY_OUTPUT" = "programs" ] || [ "$BINARY_OUTPUT" = "tests" ] ;}
+then
+  echo Place $BINARY_OUTPUT to output
+  mkdir /output/binary ||: # if exists
+  mv /build/obj-*/programs/clickhouse* /output/binary
+  if [ "$BINARY_OUTPUT" = "tests" ]
+  then
+    mv /build/obj-*/src/unit_tests_dbms /output/binary
+  fi
+fi
+ccache --show-stats ||:
+ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||:
--- a/docker/test/stateless_unbundled/Dockerfile
+++ b/docker/test/stateless_unbundled/Dockerfile
@ -0,0 +1,84 @@
+# docker build -t yandex/clickhouse-stateless-unbundled-test .
+FROM yandex/clickhouse-test-base
+
+ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"
+
+RUN apt-get --allow-unauthenticated update -y \
+    && env DEBIAN_FRONTEND=noninteractive \
+        apt-get --allow-unauthenticated install --yes --no-install-recommends \
+            alien \
+            brotli \
+            cmake \
+            devscripts \
+            expect \
+            gdb  \
+            gperf \
+            gperf \
+            heimdal-multidev \
+            intel-opencl-icd \
+            libboost-filesystem-dev \
+            libboost-iostreams-dev \
+            libboost-program-options-dev \
+            libboost-regex-dev \
+            libboost-system-dev \
+            libboost-thread-dev \
+            libc++-dev \
+            libc++abi-dev \
+            libcapnp-dev \
+            libcctz-dev \
+            libcurl4-openssl-dev \
+            libdouble-conversion-dev \
+            libgoogle-perftools-dev \
+            libhyperscan-dev \
+            libicu-dev \
+            libjemalloc-dev \
+            libldap2-dev \
+            libltdl-dev \
+            liblz4-dev \
+            libmsgpack-dev \
+            libpoco-dev \
+            libpoconetssl62 \
+            librdkafka-dev \
+            libre2-dev \
+            libreadline-dev \
+            libsasl2-dev \
+            libzstd-dev \
+            lsof \
+            moreutils \
+            ncdu \
+            netcat-openbsd \
+            ocl-icd-libopencl1 \
+            odbcinst \
+            opencl-headers \
+            openssl \
+            perl \
+            pigz \
+            pkg-config \
+            python \
+            python-lxml \
+            python-requests \
+            python-termcolor \
+            qemu-user-static \
+            sudo \
+            telnet \
+            tree \
+            tzdata \
+            unixodbc \
+            unixodbc-dev \
+            wget \
+            zlib1g-dev \
+            zookeeper \
+            zookeeperd
+
+RUN mkdir -p /tmp/clickhouse-odbc-tmp \
+   && wget --quiet -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
+   && cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \
+   && odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
+   && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
+   && rm -rf /tmp/clickhouse-odbc-tmp
+
+ENV TZ=Europe/Moscow
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+COPY run.sh /
+CMD ["/bin/bash", "/run.sh"]
--- a/docker/test/stateless_unbundled/clickhouse-statelest-test-runner.Dockerfile
+++ b/docker/test/stateless_unbundled/clickhouse-statelest-test-runner.Dockerfile
@ -0,0 +1,15 @@
+# Since right now we can't set volumes to the docker during build, we split building container in stages:
+# 1. build base container
+# 2. run base conatiner with mounted volumes
+# 3. commit container as image
+FROM ubuntu:18.10 as clickhouse-test-runner-base
+
+# A volume where directory with clickhouse packages to be mounted,
+# for later installing.
+VOLUME /packages
+
+CMD apt-get update ;\
+	DEBIAN_FRONTEND=noninteractive \
+	apt install -y /packages/clickhouse-common-static_*.deb \
+		/packages/clickhouse-client_*.deb \
+		/packages/clickhouse-test_*.deb
--- a/docker/test/stateless_unbundled/run.sh
+++ b/docker/test/stateless_unbundled/run.sh
@ -0,0 +1,65 @@
+#!/bin/bash
+
+set -e -x
+
+dpkg -i package_folder/clickhouse-common-static_*.deb
+dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
+dpkg -i package_folder/clickhouse-server_*.deb
+dpkg -i package_folder/clickhouse-client_*.deb
+dpkg -i package_folder/clickhouse-test_*.deb
+
+mkdir -p /etc/clickhouse-server/dict_examples
+ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/dict_examples/
+ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/dict_examples/
+ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/dict_examples/
+ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/
+ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/
+ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/
+ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/
+ln -s /usr/share/clickhouse-test/config/metric_log.xml /etc/clickhouse-server/config.d/
+ln -s /usr/share/clickhouse-test/config/custom_settings_prefixes.xml /etc/clickhouse-server/config.d/
+ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/
+ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/
+ln -s /usr/share/clickhouse-test/config/access_management.xml /etc/clickhouse-server/users.d/
+ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/
+ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/
+ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/
+ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/
+ln -s /usr/share/clickhouse-test/config/disks.xml /etc/clickhouse-server/config.d/
+ln -s /usr/share/clickhouse-test/config/secure_ports.xml /etc/clickhouse-server/config.d/
+ln -s /usr/share/clickhouse-test/config/clusters.xml /etc/clickhouse-server/config.d/
+ln -s /usr/share/clickhouse-test/config/graphite.xml /etc/clickhouse-server/config.d/
+ln -s /usr/share/clickhouse-test/config/server.key /etc/clickhouse-server/
+ln -s /usr/share/clickhouse-test/config/server.crt /etc/clickhouse-server/
+ln -s /usr/share/clickhouse-test/config/dhparam.pem /etc/clickhouse-server/
+
+# Retain any pre-existing config and allow ClickHouse to load it if required
+ln -s --backup=simple --suffix=_original.xml \
+    /usr/share/clickhouse-test/config/query_masking_rules.xml /etc/clickhouse-server/config.d/
+
+if [[ -n "$USE_POLYMORPHIC_PARTS" ]] && [[ "$USE_POLYMORPHIC_PARTS" -eq 1 ]]; then
+    ln -s /usr/share/clickhouse-test/config/polymorphic_parts.xml /etc/clickhouse-server/config.d/
+fi
+if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then
+    ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/
+    ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/
+fi
+
+ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml
+
+echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment
+echo "TSAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment
+echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment
+echo "ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment
+echo "UBSAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment
+echo "LLVM_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment
+
+service zookeeper start
+sleep 5
+service clickhouse-server start && sleep 5
+
+if cat /usr/bin/clickhouse-test | grep -q -- "--use-skip-list"; then
+    SKIP_LIST_OPT="--use-skip-list"
+fi
+
+clickhouse-test --testname --shard --zookeeper "$SKIP_LIST_OPT" $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt