Merge remote-tracking branch 'upstream/master' into fix15

Conflicts:
	ci/install-libraries.sh
	ci/jobs/quick-build/config

ci/README.md

@@ -1,4 +1,4 @@
-### Build and test ClickHouse on various plaforms
+## Build and test ClickHouse on various plaforms
 
 Quick and dirty scripts.
 
@@ -13,17 +13,23 @@ Another example, check build on ARM 64:
 ./run-with-docker.sh multiarch/ubuntu-core:arm64-bionic jobs/quick-build/run.sh
 ```
 
-Look at `default_config` and `jobs/quick-build/config`
+Another example, check build on FreeBSD:
+```
+./prepare-vagrant-image-freebsd.sh
+./run-with-vagrant.sh freebsd jobs/quick-build/run.sh
+```
+
+Look at `default_config` and `jobs/quick-build/run.sh`
 
 Various possible options. We are not going to automate testing all of them.
 
-##### CPU architectures:
+#### CPU architectures:
 - x86_64;
 - AArch64.
 
 x86_64 is the main CPU architecture. We also have minimal support for AArch64.
 
-##### Operating systems:
+#### Operating systems:
 - Linux;
 - FreeBSD.
 
@@ -31,7 +37,7 @@ We also target Mac OS X, but it's more difficult to test.
 Linux is the main. FreeBSD is also supported as production OS.  
 Mac OS is intended only for development and have minimal support: client should work, server should just start.
 
-##### Linux distributions:
+#### Linux distributions:
 For build:
 - Ubuntu Bionic;
 - Ubuntu Trusty.
@@ -42,83 +48,83 @@ For run:
 
 We should support almost any Linux to run ClickHouse. That's why we test also on old distributions.
 
-##### How to obtain sources:
+#### How to obtain sources:
 - use sources from local working copy;
 - clone sources from github;
 - download source tarball.
 
-##### Compilers:
+#### Compilers:
 - gcc-7;
 - gcc-8;
 - clang-6;
 - clang-svn.
 
-##### Compiler installation:
+#### Compiler installation:
 - from OS packages;
 - build from sources.
 
-##### C++ standard library implementation:
+#### C++ standard library implementation:
 - libc++;
 - libstdc++ with C++11 ABI;
 - libstdc++ with old ABI.
 
 When building with clang, libc++ is used. When building with gcc, we choose libstdc++ with C++11 ABI.
 
-##### Linkers:
+#### Linkers:
 - ldd;
 - gold;
 
 When building with clang on x86_64, ldd is used. Otherwise we use gold.
 
-##### Build types:
+#### Build types:
 - RelWithDebInfo;
 - Debug;
 - ASan;
 - TSan.
 
-##### Build types, extra:
+#### Build types, extra:
 - -g0 for quick build;
 - enable test coverage;
 - debug tcmalloc.
 
-##### What to build:
+#### What to build:
 - only `clickhouse` target;
 - all targets;
 - debian packages;
 
 We also have intent to build RPM and simple tgz packages.
 
-##### Where to get third-party libraries:
+#### Where to get third-party libraries:
 - from contrib directory (submodules);
 - from OS packages.
 
 The only production option is to use libraries from contrib directory.  
 Using libraries from OS packages is discouraged, but we also support this option.
 
-##### Linkage types:
+#### Linkage types:
 - static;
 - shared;
 
 Static linking is the only option for production usage.  
 We also have support for shared linking, but it is indended only for developers.
 
-##### Make tools:
+#### Make tools:
 - make;
 - ninja.
 
-##### Installation options:
+#### Installation options:
 - run built `clickhouse` binary directly;
 - install from packages.
 
-##### How to obtain packages:
+#### How to obtain packages:
 - build them;
 - download from repository.
 
-##### Sanity checks:
+#### Sanity checks:
 - check that clickhouse binary has no dependencies on unexpected shared libraries;
 - check that source code have no style violations.
 
-##### Tests:
+#### Tests:
 - Functional tests;
 - Integration tests;
 - Unit tests;
@@ -127,10 +133,10 @@ We also have support for shared linking, but it is indended only for developers.
 - Tests for external dictionaries (should be moved to integration tests);
 - Jepsen like tests for quorum inserts (not yet available in opensource).
 
-##### Tests extra:
+#### Tests extra:
 - Run functional tests with Valgrind.
 
-##### Static analyzers:
+#### Static analyzers:
 - CppCheck;
 - clang-tidy;
 - Coverity.

ci/build-clang-from-sources.sh

@@ -4,8 +4,8 @@ set -e -x
 source default-config
 
 # TODO Non debian systems
-$SUDO apt-get install -y subversion
+./install-os-packages.sh svn
-apt-cache search cmake3 | grep -P '^cmake3 ' && $SUDO apt-get -y install cmake3 || $SUDO apt-get -y install cmake
+./install-os-packages.sh cmake
 
 mkdir "${WORKSPACE}/llvm"
 

ci/build-gcc-from-sources.sh

@@ -3,7 +3,7 @@ set -e -x
 
 source default-config
 
-$SUDO apt-get install -y curl
+./install-os-packages.sh curl
 
 if [[ "${GCC_SOURCES_VERSION}" == "latest" ]]; then
     GCC_SOURCES_VERSION=$(curl -sSL https://ftpmirror.gnu.org/gcc/ | grep -oE 'gcc-[0-9]+(\.[0-9]+)+' | sort -Vr | head -n1)

ci/check-syntax.sh

@@ -3,7 +3,7 @@ set -e -x
 
 source default-config
 
-$SUDO apt-get install -y jq
+./install-os-packages.sh jq
 
 [[ -d "${WORKSPACE}/sources" ]] || die "Run get-sources.sh first"
 

ci/default-config

@@ -9,7 +9,7 @@ SCRIPTPATH=$(pwd)
 WORKSPACE=${SCRIPTPATH}/workspace
 PROJECT_ROOT=$(cd $SCRIPTPATH/.. && pwd)
 
-# All scripts take no arguments. All arguments must be in config.
+# Almost all scripts take no arguments. Arguments should be in config.
 
 # get-sources
 SOURCES_METHOD=local                # clone, local, tarball
@@ -55,7 +55,7 @@ function die {
 
 [[ $EUID -ne 0 ]] && SUDO=sudo
 
-command -v apt-get && $SUDO apt-get update
+./install-os-packages.sh prepare
 
 # Configuration parameters may be overriden with CONFIG environment variable pointing to config file.
 [[ -n "$CONFIG" ]] && source $CONFIG

ci/get-sources.sh

@@ -4,12 +4,12 @@ set -e -x
 source default-config
 
 if [[ "$SOURCES_METHOD" == "clone" ]]; then
-    $SUDO apt-get install -y git
+    ./install-os-packages.sh git
     SOURCES_DIR="${WORKSPACE}/sources"
     mkdir -p "${SOURCES_DIR}"
     git clone --recursive --branch "$SOURCES_BRANCH" "$SOURCES_CLONE_URL" "${SOURCES_DIR}"
     pushd "${SOURCES_DIR}"
-    git checkout "$SOURCES_COMMIT"
+    git checkout --recurse-submodules "$SOURCES_COMMIT"
     popd
 elif [[ "$SOURCES_METHOD" == "local" ]]; then
     ln -f -s "${PROJECT_ROOT}" "${WORKSPACE}/sources"

ci/install-compiler-from-packages.sh

@@ -3,27 +3,20 @@ set -e -x
 
 source default-config
 
-# TODO Non debian systems
 # TODO Install from PPA on older Ubuntu
 
-if [ -f '/etc/lsb-release' ]; then
+./install-os-packages.sh ${COMPILER}-${COMPILER_PACKAGE_VERSION}
-    source /etc/lsb-release
+
-    if [[ "$DISTRIB_ID" == "Ubuntu" ]]; then
+if [[ "$COMPILER" == "gcc" ]]; then
-        if [[ "$COMPILER" == "gcc" ]]; then
+    if command -v gcc-${COMPILER_PACKAGE_VERSION}; then export CC=gcc-${COMPILER_PACKAGE_VERSION} CXX=g++-${COMPILER_PACKAGE_VERSION};
-            $SUDO apt-get -y install gcc-${COMPILER_PACKAGE_VERSION} g++-${COMPILER_PACKAGE_VERSION}
+    elif command -v gcc${COMPILER_PACKAGE_VERSION}; then export CC=gcc${COMPILER_PACKAGE_VERSION} CXX=g++${COMPILER_PACKAGE_VERSION};
-            export CC=gcc-${COMPILER_PACKAGE_VERSION}
+    elif command -v gcc; then export CC=gcc CXX=g++;
-            export CXX=g++-${COMPILER_PACKAGE_VERSION}
+    fi
-        elif [[ "$COMPILER" == "clang" ]]; then
+elif [[ "$COMPILER" == "clang" ]]; then
-            [[ $(uname -m) == "x86_64" ]] && LLD="lld-${COMPILER_PACKAGE_VERSION}"
+    if command -v clang-${COMPILER_PACKAGE_VERSION}; then export CC=clang-${COMPILER_PACKAGE_VERSION} CXX=clang++-${COMPILER_PACKAGE_VERSION};
-            $SUDO apt-get -y install clang-${COMPILER_PACKAGE_VERSION} "$LLD" libc++-dev libc++abi-dev
+    elif command -v clang${COMPILER_PACKAGE_VERSION}; then export CC=clang${COMPILER_PACKAGE_VERSION} CXX=clang++${COMPILER_PACKAGE_VERSION};
-            export CC=clang-${COMPILER_PACKAGE_VERSION}
+    elif command -v clang; then export CC=clang CXX=clang++;
-            export CXX=clang++-${COMPILER_PACKAGE_VERSION}
-        else
-            die "Unknown compiler specified"
-        fi
-    else
-        die "Unknown Linux variant"
     fi
 else
-    die "Unknown OS"
+    die "Unknown compiler specified"
 fi

ci/install-libraries.sh

@@ -3,10 +3,12 @@ set -e -x
 
 source default-config
 
-# TODO Non-debian systems
+./install-os-packages.sh libssl-dev
-
+./install-os-packages.sh libicu-dev
-$SUDO apt-get -y install libssl-dev libicu-dev libreadline-dev libmariadbclient-dev unixodbc-dev ccache
+./install-os-packages.sh libreadline-dev
+./install-os-packages.sh libmariadbclient-dev
+./install-os-packages.sh libunixodbc-dev
 
 if [[ "$ENABLE_EMBEDDED_COMPILER" == 1 && "$USE_LLVM_LIBRARIES_FROM_SYSTEM" == 1 ]]; then
-    $SUDO apt-get -y install liblld-5.0-dev libclang-5.0-dev
+    ./install-os-packages.sh llvm-libs-5.0
 fi

ci/install-os-packages.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+set -e -x
+
+# Dispatches package installation on various OS and distributives
+
+WHAT=$1
+
+[[ $EUID -ne 0 ]] && SUDO=sudo
+
+command -v apt-get && PACKAGE_MANAGER=apt
+command -v yum && PACKAGE_MANAGER=yum
+command -v pkg && PACKAGE_MANAGER=pkg
+
+
+case $PACKAGE_MANAGER in
+    apt)
+        case $WHAT in
+            prepare)
+                $SUDO apt-get update
+                ;;
+            svn)
+                $SUDO apt-get install -y subversion
+                ;;
+            gcc*)
+                $SUDO apt-get install -y $WHAT ${WHAT/cc/++}
+                ;;
+            clang*)
+                $SUDO apt-get install -y $WHAT libc++-dev libc++abi-dev
+                [[ $(uname -m) == "x86_64" ]] && $SUDO apt-get install -y ${WHAT/clang/lld}
+                ;;
+            git)
+                $SUDO apt-get install -y git
+                ;;
+            cmake)
+                $SUDO apt-get install -y cmake3 || $SUDO apt-get install -y cmake
+                ;;
+            curl)
+                $SUDO apt-get install -y curl
+                ;;
+            jq)
+                $SUDO apt-get install -y jq
+                ;;
+            libssl-dev)
+                $SUDO apt-get install -y libssl-dev
+                ;;
+            libicu-dev)
+                $SUDO apt-get install -y libicu-dev
+                ;;
+            libreadline-dev)
+                $SUDO apt-get install -y libreadline-dev
+                ;;
+            libunixodbc-dev)
+                $SUDO apt-get install -y unixodbc-dev
+                ;;
+            libmysqlclient-dev)
+                $SUDO apt-get install -y libmysqlclient-dev
+                ;;
+            llvm-libs*)
+                $SUDO apt-get -y install ${WHAT/llvm-libs/liblld}-dev ${WHAT/llvm-libs/libclang}-dev
+                ;;
+            qemu-user-static)
+                $SUDO apt-get install -y quemu-user-static
+                ;;
+            vagrant-virtualbox)
+                $SUDO apt-get install -y vagrant virtualbox
+                ;;
+            *)
+                echo "Unknown package"; exit 1;
+                ;;
+        esac
+        ;;
+    pkg)
+        case $WHAT in
+            prepare)
+                ;;
+            svn)
+                $SUDO pkg install -y subversion
+                ;;
+            gcc*)
+                $SUDO pkg install -y ${WHAT/-/}
+                ;;
+            clang*)
+                $SUDO pkg install -y clang-devel
+                ;;
+            git)
+                $SUDO pkg install -y git
+                ;;
+            cmake)
+                $SUDO pkg install -y cmake
+                ;;
+            curl)
+                $SUDO pkg install -y curl
+                ;;
+            jq)
+                $SUDO pkg install -y jq
+                ;;
+            libssl-dev)
+                $SUDO pkg install -y openssl
+                ;;
+            libicu-dev)
+                $SUDO pkg install -y icu
+                ;;
+            libreadline-dev)
+                $SUDO pkg install -y readline
+                ;;
+            libunixodbc-dev)
+                $SUDO pkg install -y unixODBC
+                ;;
+            libmysqlclient-dev)
+                $SUDO pkg install -y mysql57-client
+                ;;
+            *)
+                echo "Unknown package"; exit 1;
+                ;;
+        esac
+        ;;
+    *)
+        echo "Unknown distributive"; exit 1;
+        ;;
+esac

ci/jobs/quick-build/README.md

@@ -0,0 +1,5 @@
+## Build with debug mode and without many libraries
+
+This job is intended as first check that build is not broken on wide variety of platforms. 
+
+Results of this build are not intended for production usage.

ci/jobs/quick-build/config

@@ -1,11 +0,0 @@
-SOURCES_METHOD=local
-COMPILER=clang
-COMPILER_INSTALL_METHOD=packages
-COMPILER_PACKAGE_VERSION=6.0
-USE_LLVM_LIBRARIES_FROM_SYSTEM=0
-BUILD_METHOD=normal
-BUILD_TARGETS=clickhouse
-BUILD_TYPE=Debug
-ENABLE_EMBEDDED_COMPILER=0
-CMAKE_FLAGS="-D CMAKE_C_FLAGS_ADD=-g0 -D CMAKE_CXX_FLAGS_ADD=-g0 -D ENABLE_TCMALLOC=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_UNWIND=0 -D ENABLE_ICU=0"
-CMAKE_FLAGS+=" -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_MYSQL=0 -D ENABLE_POCO_ODBC=0"

ci/jobs/quick-build/run.sh

@@ -12,6 +12,20 @@ cd "$(dirname $0)"/../..
 
 . default-config
 
+SOURCES_METHOD=local
+COMPILER=clang
+COMPILER_INSTALL_METHOD=packages
+COMPILER_PACKAGE_VERSION=6.0
+USE_LLVM_LIBRARIES_FROM_SYSTEM=0
+BUILD_METHOD=normal
+BUILD_TARGETS=clickhouse
+BUILD_TYPE=Debug
+ENABLE_EMBEDDED_COMPILER=0
+
+CMAKE_FLAGS="-D CMAKE_C_FLAGS_ADD=-g0 -D CMAKE_CXX_FLAGS_ADD=-g0 -D ENABLE_TCMALLOC=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_UNWIND=0 -D ENABLE_ICU=0 -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_POCO_ODBC=0 -D ENABLE_MYSQL=0"
+
+[[ $(uname) == "FreeBSD" ]] && COMPILER_PACKAGE_VERSION=devel
+
 . get-sources.sh
 . prepare-toolchain.sh
 . install-libraries.sh

ci/prepare-docker-image-ubuntu.sh

@@ -6,7 +6,7 @@ source default-config
 ./check-docker.sh
 
 # http://fl47l1n3.net/2015/12/24/binfmt/
-$SUDO apt-get -y install qemu-user-static
+./install-os-packages.sh qemu-user-static
 
 pushd docker-multiarch
 

ci/prepare-toolchain.sh

@@ -3,11 +3,10 @@ set -e -x
 
 source default-config
 
-# TODO Non debian systems
+./install-os-packages.sh cmake
-apt-cache search cmake3 | grep -P '^cmake3 ' && $SUDO apt-get -y install cmake3 || $SUDO apt-get -y install cmake
 
 if [[ "$COMPILER_INSTALL_METHOD" == "packages" ]]; then
-    . install-compiler-from-packages.sh;
+    . install-compiler-from-packages.sh
 elif [[ "$COMPILER_INSTALL_METHOD" == "sources" ]]; then
     . install-compiler-from-sources.sh
 else

ci/prepare-vagrant-image-freebsd.sh

@@ -3,7 +3,7 @@ set -e -x
 
 source default-config
 
-$SUDO apt-get -y install vagrant virtualbox
+./install-os-packages.sh vagrant-virtualbox
 
 pushd "vagrant-freebsd"
 vagrant up

ci/run-with-vagrant.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e -x
+
+[[ -r "vagrant-${1}/vagrant-ssh" ]] || die "Run prepare-vagrant-image-... first."
+
+pushd vagrant-$1
+
+shopt -s extglob
+vagrant ssh -c "mkdir ClickHouse"
+scp -F vagrant-ssh -r ../../../ClickHouse/!(*build*) default:~/ClickHouse
+vagrant ssh -c "cd ClickHouse/ci; $2"
+
+popd