Docker update and improve (#2450)

* Fix user name on distributed queries

* wip

* fix

* Docker update and improve

* Fix

docker/builder/Dockerfile

@@ -1,11 +1,11 @@
-FROM ubuntu:17.10
+FROM ubuntu:18.04
 
 RUN apt-get update -y && \
-    apt-get install -y \
+    env DEBIAN_FRONTEND=noninteractive apt-get install -y \
-        cmake pkg-config gcc-7 g++-7 \
+        cmake ninja-build ccache pkg-config gcc g++ \
-        liblld-5.0-dev libclang-5.0-dev \
+        liblld-6.0-dev libclang-6.0-dev \
-        libssl-dev libicu-dev libreadline-dev libmysqlclient-dev unixodbc-dev
+        libssl-dev libicu-dev libreadline-dev libmysqlclient-dev unixodbc-dev \
-        # For tests: bash expect python python-lxml python-termcolor curl perl sudo tzdata
+        bash expect python python-lxml python-termcolor python-requests curl perl sudo tzdata
 
 ADD build.sh /
 RUN chmod +x /build.sh

docker/builder/Makefile

@@ -1,5 +1,6 @@
-build:
+build: image
-	docker run --network=host --rm --workdir /server --volume $(realpath ../..):/server -it yandex/clickhouse-builder
+	mkdir -p $(HOME)/.ccache
+	docker run --network=host --rm --workdir /server --volume $(realpath ../..):/server --mount=type=bind,source=$(HOME)/.ccache,destination=/ccache -e CCACHE_DIR=/ccache -it yandex/clickhouse-builder
 
 pull:
 	docker pull yandex/clickhouse-builder

docker/builder/build.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
+#ccache -s
 mkdir -p /server/build_docker
 cd /server/build_docker
-cmake /server -D ENABLE_TESTS=0
+cmake -G Ninja /server -DENABLE_TESTS=1
-make -j $(nproc || grep -c ^processor /proc/cpuinfo)
+cmake --build .
-#ctest -V -j $(nproc || grep -c ^processor /proc/cpuinfo)
+env TEST_OPT="--skip long compile $TEST_OPT" ctest -V -j $(nproc || grep -c ^processor /proc/cpuinfo)

docker/client/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:17.10
+FROM ubuntu:18.04
 
 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
 ARG version=\*
@@ -9,7 +9,7 @@ RUN apt-get update && \
     apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 && \
     echo $repository | tee /etc/apt/sources.list.d/clickhouse.list && \
     apt-get update && \
-    apt-get install --allow-unauthenticated -y clickhouse-client=$version locales tzdata && \
+    env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-client=$version locales tzdata && \
     rm -rf /var/lib/apt/lists/* /var/cache/debconf && \
     apt-get clean
 

docker/server/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:17.10
+FROM ubuntu:18.04
 
 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
 ARG version=\*
@@ -9,7 +9,7 @@ RUN apt-get update && \
     apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 && \
     echo $repository | tee /etc/apt/sources.list.d/clickhouse.list && \
     apt-get update && \
-    apt-get install --allow-unauthenticated -y "clickhouse-server=$version" libgcc-7-dev && \
+    env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y "clickhouse-server=$version" libgcc-7-dev && \
     rm -rf /var/lib/apt/lists/* /var/cache/debconf && \
     apt-get clean
 

docker/test/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:17.10
+FROM ubuntu:18.04
 
 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
 ARG version=\*
@@ -9,11 +9,8 @@ RUN apt-get update && \
     apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 && \
     echo $repository | tee /etc/apt/sources.list.d/clickhouse.list && \
     apt-get update && \
-    apt-get install --allow-unauthenticated -y clickhouse-test && \
+    env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-test && \
     rm -rf /var/lib/apt/lists/* /var/cache/debconf && \
     apt-get clean
 
-# clickhouse-test bug: it doesn't start without server config, remove after release 1.1.54372 :
-RUN mkdir -p /etc/clickhouse-server && echo "<y></y>" > /etc/clickhouse-server/config.xml
-
 ENTRYPOINT ["/usr/bin/clickhouse-test"]

release

@@ -53,7 +53,7 @@ do
         shift
     elif [[ $1 == '--fast' ]]; then
         # Wrong but fast pbuilder mode: create base package with all depends
-        EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake ninja-build gcc-7 g++-7 libc6-dev libmariadbclient-dev libicu-dev libltdl-dev libreadline-dev libssl-dev unixodbc-dev psmisc bash expect python python-lxml python-termcolor curl perl sudo openssl"
+        EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake ninja-build gcc-7 g++-7 libc6-dev libmariadbclient-dev libicu-dev libltdl-dev libreadline-dev libssl-dev unixodbc-dev psmisc bash expect python python-lxml python-termcolor python-requests curl perl sudo openssl"
         shift
     else
         echo "Unknown option $1"

utils/build/build_debian.sh

@@ -11,7 +11,7 @@ sudo apt install -y git bash cmake gcc-7 g++-7 libicu-dev libreadline-dev libmys
 #sudo apt install -y libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libzstd-dev libre2-dev libsparsehash-dev librdkafka-dev libcapnp-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libunwind-dev googletest libcctz-dev
 
 # install testing only stuff if you want:
-sudo apt install -y python python-lxml python-termcolor curl perl
+sudo apt install -y python python-lxml python-termcolor python-requests curl perl
 
 # Checkout ClickHouse sources
 git clone --recursive https://github.com/yandex/ClickHouse.git

utils/build/build_freebsd.sh

@@ -22,7 +22,7 @@
 sudo pkg install devel/git devel/cmake shells/bash devel/icu devel/libltdl databases/unixODBC devel/google-perftools devel/libdouble-conversion archivers/zstd archivers/liblz4 devel/sparsehash devel/re2
 
 #  install testing only stuff if you want:
-sudo pkg install lang/python devel/py-lxml devel/py-termcolor ftp/curl perl5
+sudo pkg install lang/python devel/py-lxml devel/py-termcolor www/py-requests ftp/curl perl5
 
 #  If you want ODBC support: Check UNIXODBC option:
 # make -C /usr/ports/devel/poco config reinstall