From a98d33f77fca314e7459742d1740b56e1b302e65 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 14 May 2018 21:49:51 +0300 Subject: [PATCH] CI: added support to build on FreeBSD under Vagrant [#CLICKHOUSE-2] --- ci/README.md | 50 ++++++----- ci/build-clang-from-sources.sh | 4 +- ci/build-gcc-from-sources.sh | 2 +- ci/check-syntax.sh | 2 +- ci/default-config | 4 +- ci/get-sources.sh | 4 +- ci/install-compiler-from-packages.sh | 31 +++---- ci/install-libraries.sh | 10 ++- ci/install-os-packages.sh | 120 +++++++++++++++++++++++++++ ci/jobs/quick-build/README.md | 5 ++ ci/jobs/quick-build/config | 12 --- ci/jobs/quick-build/run.sh | 15 ++++ ci/prepare-docker-image-ubuntu.sh | 2 +- ci/prepare-toolchain.sh | 5 +- ci/prepare-vagrant-image-freebsd.sh | 2 +- ci/run-with-vagrant.sh | 13 +++ 16 files changed, 211 insertions(+), 70 deletions(-) create mode 100755 ci/install-os-packages.sh create mode 100644 ci/jobs/quick-build/README.md delete mode 100644 ci/jobs/quick-build/config create mode 100755 ci/run-with-vagrant.sh diff --git a/ci/README.md b/ci/README.md index 6eeb35c1c25..733cbce80c9 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,4 +1,4 @@ -### Build and test ClickHouse on various plaforms +## Build and test ClickHouse on various plaforms Quick and dirty scripts. @@ -13,17 +13,23 @@ Another example, check build on ARM 64: ./run-with-docker.sh multiarch/ubuntu-core:arm64-bionic jobs/quick-build/run.sh ``` -Look at `default_config` and `jobs/quick-build/config` +Another example, check build on FreeBSD: +``` +./prepare-vagrant-image-freebsd.sh +./run-with-vagrant.sh freebsd jobs/quick-build/run.sh +``` + +Look at `default_config` and `jobs/quick-build/run.sh` Various possible options. We are not going to automate testing all of them. -##### CPU architectures: +#### CPU architectures: - x86_64; - AArch64. x86_64 is the main CPU architecture. We also have minimal support for AArch64. -##### Operating systems: +#### Operating systems: - Linux; - FreeBSD. @@ -31,7 +37,7 @@ We also target Mac OS X, but it's more difficult to test. Linux is the main. FreeBSD is also supported as production OS. Mac OS is intended only for development and have minimal support: client should work, server should just start. -##### Linux distributions: +#### Linux distributions: For build: - Ubuntu Bionic; - Ubuntu Trusty. @@ -42,83 +48,83 @@ For run: We should support almost any Linux to run ClickHouse. That's why we test also on old distributions. -##### How to obtain sources: +#### How to obtain sources: - use sources from local working copy; - clone sources from github; - download source tarball. -##### Compilers: +#### Compilers: - gcc-7; - gcc-8; - clang-6; - clang-svn. -##### Compiler installation: +#### Compiler installation: - from OS packages; - build from sources. -##### C++ standard library implementation: +#### C++ standard library implementation: - libc++; - libstdc++ with C++11 ABI; - libstdc++ with old ABI. When building with clang, libc++ is used. When building with gcc, we choose libstdc++ with C++11 ABI. -##### Linkers: +#### Linkers: - ldd; - gold; When building with clang on x86_64, ldd is used. Otherwise we use gold. -##### Build types: +#### Build types: - RelWithDebInfo; - Debug; - ASan; - TSan. -##### Build types, extra: +#### Build types, extra: - -g0 for quick build; - enable test coverage; - debug tcmalloc. -##### What to build: +#### What to build: - only `clickhouse` target; - all targets; - debian packages; We also have intent to build RPM and simple tgz packages. -##### Where to get third-party libraries: +#### Where to get third-party libraries: - from contrib directory (submodules); - from OS packages. The only production option is to use libraries from contrib directory. Using libraries from OS packages is discouraged, but we also support this option. -##### Linkage types: +#### Linkage types: - static; - shared; Static linking is the only option for production usage. We also have support for shared linking, but it is indended only for developers. -##### Make tools: +#### Make tools: - make; - ninja. -##### Installation options: +#### Installation options: - run built `clickhouse` binary directly; - install from packages. -##### How to obtain packages: +#### How to obtain packages: - build them; - download from repository. -##### Sanity checks: +#### Sanity checks: - check that clickhouse binary has no dependencies on unexpected shared libraries; - check that source code have no style violations. -##### Tests: +#### Tests: - Functional tests; - Integration tests; - Unit tests; @@ -127,10 +133,10 @@ We also have support for shared linking, but it is indended only for developers. - Tests for external dictionaries (should be moved to integration tests); - Jepsen like tests for quorum inserts (not yet available in opensource). -##### Tests extra: +#### Tests extra: - Run functional tests with Valgrind. -##### Static analyzers: +#### Static analyzers: - CppCheck; - clang-tidy; - Coverity. diff --git a/ci/build-clang-from-sources.sh b/ci/build-clang-from-sources.sh index 64898c5fdc3..7e3793c8148 100755 --- a/ci/build-clang-from-sources.sh +++ b/ci/build-clang-from-sources.sh @@ -4,8 +4,8 @@ set -e -x source default-config # TODO Non debian systems -$SUDO apt-get install -y subversion -apt-cache search cmake3 | grep -P '^cmake3 ' && $SUDO apt-get -y install cmake3 || $SUDO apt-get -y install cmake +./install-os-packages.sh svn +./install-os-packages.sh cmake mkdir "${WORKSPACE}/llvm" diff --git a/ci/build-gcc-from-sources.sh b/ci/build-gcc-from-sources.sh index b41ac0365bd..0734b22335a 100755 --- a/ci/build-gcc-from-sources.sh +++ b/ci/build-gcc-from-sources.sh @@ -3,7 +3,7 @@ set -e -x source default-config -$SUDO apt-get install -y curl +./install-os-packages.sh curl if [[ "${GCC_SOURCES_VERSION}" == "latest" ]]; then GCC_SOURCES_VERSION=$(curl -sSL https://ftpmirror.gnu.org/gcc/ | grep -oE 'gcc-[0-9]+(\.[0-9]+)+' | sort -Vr | head -n1) diff --git a/ci/check-syntax.sh b/ci/check-syntax.sh index c5043ff512c..e95e38346d6 100755 --- a/ci/check-syntax.sh +++ b/ci/check-syntax.sh @@ -3,7 +3,7 @@ set -e -x source default-config -$SUDO apt-get install -y jq +./install-os-packages.sh jq [[ -d "${WORKSPACE}/sources" ]] || die "Run get-sources.sh first" diff --git a/ci/default-config b/ci/default-config index 7837b1fe57d..7cc22758d54 100644 --- a/ci/default-config +++ b/ci/default-config @@ -9,7 +9,7 @@ SCRIPTPATH=$(pwd) WORKSPACE=${SCRIPTPATH}/workspace PROJECT_ROOT=$(cd $SCRIPTPATH/.. && pwd) -# All scripts take no arguments. All arguments must be in config. +# Almost all scripts take no arguments. Arguments should be in config. # get-sources SOURCES_METHOD=local # clone, local, tarball @@ -55,7 +55,7 @@ function die { [[ $EUID -ne 0 ]] && SUDO=sudo -command -v apt-get && $SUDO apt-get update +./install-os-packages.sh prepare # Configuration parameters may be overriden with CONFIG environment variable pointing to config file. [[ -n "$CONFIG" ]] && source $CONFIG diff --git a/ci/get-sources.sh b/ci/get-sources.sh index f09f8c3c812..ee57b0ec27d 100755 --- a/ci/get-sources.sh +++ b/ci/get-sources.sh @@ -4,12 +4,12 @@ set -e -x source default-config if [[ "$SOURCES_METHOD" == "clone" ]]; then - $SUDO apt-get install -y git + ./install-os-packages.sh git SOURCES_DIR="${WORKSPACE}/sources" mkdir -p "${SOURCES_DIR}" git clone --recursive --branch "$SOURCES_BRANCH" "$SOURCES_CLONE_URL" "${SOURCES_DIR}" pushd "${SOURCES_DIR}" - git checkout "$SOURCES_COMMIT" + git checkout --recurse-submodules "$SOURCES_COMMIT" popd elif [[ "$SOURCES_METHOD" == "local" ]]; then ln -f -s "${PROJECT_ROOT}" "${WORKSPACE}/sources" diff --git a/ci/install-compiler-from-packages.sh b/ci/install-compiler-from-packages.sh index c46f09219e7..53909435a06 100755 --- a/ci/install-compiler-from-packages.sh +++ b/ci/install-compiler-from-packages.sh @@ -3,27 +3,20 @@ set -e -x source default-config -# TODO Non debian systems # TODO Install from PPA on older Ubuntu -if [ -f '/etc/lsb-release' ]; then - source /etc/lsb-release - if [[ "$DISTRIB_ID" == "Ubuntu" ]]; then - if [[ "$COMPILER" == "gcc" ]]; then - $SUDO apt-get -y install gcc-${COMPILER_PACKAGE_VERSION} g++-${COMPILER_PACKAGE_VERSION} - export CC=gcc-${COMPILER_PACKAGE_VERSION} - export CXX=g++-${COMPILER_PACKAGE_VERSION} - elif [[ "$COMPILER" == "clang" ]]; then - [[ $(uname -m) == "x86_64" ]] && LLD="lld-${COMPILER_PACKAGE_VERSION}" - $SUDO apt-get -y install clang-${COMPILER_PACKAGE_VERSION} "$LLD" libc++-dev libc++abi-dev - export CC=clang-${COMPILER_PACKAGE_VERSION} - export CXX=clang++-${COMPILER_PACKAGE_VERSION} - else - die "Unknown compiler specified" - fi - else - die "Unknown Linux variant" +./install-os-packages.sh ${COMPILER}-${COMPILER_PACKAGE_VERSION} + +if [[ "$COMPILER" == "gcc" ]]; then + if command -v gcc-${COMPILER_PACKAGE_VERSION}; then export CC=gcc-${COMPILER_PACKAGE_VERSION} CXX=g++-${COMPILER_PACKAGE_VERSION}; + elif command -v gcc${COMPILER_PACKAGE_VERSION}; then export CC=gcc${COMPILER_PACKAGE_VERSION} CXX=g++${COMPILER_PACKAGE_VERSION}; + elif command -v gcc; then export CC=gcc CXX=g++; + fi +elif [[ "$COMPILER" == "clang" ]]; then + if command -v clang-${COMPILER_PACKAGE_VERSION}; then export CC=clang-${COMPILER_PACKAGE_VERSION} CXX=clang++-${COMPILER_PACKAGE_VERSION}; + elif command -v clang${COMPILER_PACKAGE_VERSION}; then export CC=clang${COMPILER_PACKAGE_VERSION} CXX=clang++${COMPILER_PACKAGE_VERSION}; + elif command -v clang; then export CC=clang CXX=clang++; fi else - die "Unknown OS" + die "Unknown compiler specified" fi diff --git a/ci/install-libraries.sh b/ci/install-libraries.sh index 7070083d57e..8f881c2db61 100755 --- a/ci/install-libraries.sh +++ b/ci/install-libraries.sh @@ -3,10 +3,12 @@ set -e -x source default-config -# TODO Non-debian systems - -$SUDO apt-get -y install libssl-dev libicu-dev libreadline-dev libmysqlclient-dev unixodbc-dev +./install-os-packages.sh libssl-dev +./install-os-packages.sh libicu-dev +./install-os-packages.sh libreadline-dev +./install-os-packages.sh libmysqlclient-dev +./install-os-packages.sh libunixodbc-dev if [[ "$ENABLE_EMBEDDED_COMPILER" == 1 && "$USE_LLVM_LIBRARIES_FROM_SYSTEM" == 1 ]]; then - $SUDO apt-get -y install liblld-5.0-dev libclang-5.0-dev + ./install-os-packages.sh llvm-libs-5.0 fi diff --git a/ci/install-os-packages.sh b/ci/install-os-packages.sh new file mode 100755 index 00000000000..46502094507 --- /dev/null +++ b/ci/install-os-packages.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +set -e -x + +# Dispatches package installation on various OS and distributives + +WHAT=$1 + +[[ $EUID -ne 0 ]] && SUDO=sudo + +command -v apt-get && PACKAGE_MANAGER=apt +command -v yum && PACKAGE_MANAGER=yum +command -v pkg && PACKAGE_MANAGER=pkg + + +case $PACKAGE_MANAGER in + apt) + case $WHAT in + prepare) + $SUDO apt-get update + ;; + svn) + $SUDO apt-get install -y subversion + ;; + gcc*) + $SUDO apt-get install -y $WHAT ${WHAT/cc/++} + ;; + clang*) + $SUDO apt-get install -y $WHAT libc++-dev libc++abi-dev + [[ $(uname -m) == "x86_64" ]] && $SUDO apt-get install -y ${WHAT/clang/lld} + ;; + git) + $SUDO apt-get install -y git + ;; + cmake) + $SUDO apt-get install -y cmake3 || $SUDO apt-get install -y cmake + ;; + curl) + $SUDO apt-get install -y curl + ;; + jq) + $SUDO apt-get install -y jq + ;; + libssl-dev) + $SUDO apt-get install -y libssl-dev + ;; + libicu-dev) + $SUDO apt-get install -y libicu-dev + ;; + libreadline-dev) + $SUDO apt-get install -y libreadline-dev + ;; + libunixodbc-dev) + $SUDO apt-get install -y unixodbc-dev + ;; + libmysqlclient-dev) + $SUDO apt-get install -y libmysqlclient-dev + ;; + llvm-libs*) + $SUDO apt-get -y install ${WHAT/llvm-libs/liblld}-dev ${WHAT/llvm-libs/libclang}-dev + ;; + qemu-user-static) + $SUDO apt-get install -y quemu-user-static + ;; + vagrant-virtualbox) + $SUDO apt-get install -y vagrant virtualbox + ;; + *) + echo "Unknown package"; exit 1; + ;; + esac + ;; + pkg) + case $WHAT in + prepare) + ;; + svn) + $SUDO pkg install -y subversion + ;; + gcc*) + $SUDO pkg install -y ${WHAT/-/} + ;; + clang*) + $SUDO pkg install -y clang-devel + ;; + git) + $SUDO pkg install -y git + ;; + cmake) + $SUDO pkg install -y cmake + ;; + curl) + $SUDO pkg install -y curl + ;; + jq) + $SUDO pkg install -y jq + ;; + libssl-dev) + $SUDO pkg install -y openssl + ;; + libicu-dev) + $SUDO pkg install -y icu + ;; + libreadline-dev) + $SUDO pkg install -y readline + ;; + libunixodbc-dev) + $SUDO pkg install -y unixODBC + ;; + libmysqlclient-dev) + $SUDO pkg install -y mysql57-client + ;; + *) + echo "Unknown package"; exit 1; + ;; + esac + ;; + *) + echo "Unknown distributive"; exit 1; + ;; +esac diff --git a/ci/jobs/quick-build/README.md b/ci/jobs/quick-build/README.md new file mode 100644 index 00000000000..803acae0f93 --- /dev/null +++ b/ci/jobs/quick-build/README.md @@ -0,0 +1,5 @@ +## Build with debug mode and without many libraries + +This job is intended as first check that build is not broken on wide variety of platforms. + +Results of this build are not intended for production usage. diff --git a/ci/jobs/quick-build/config b/ci/jobs/quick-build/config deleted file mode 100644 index c45d9690c7a..00000000000 --- a/ci/jobs/quick-build/config +++ /dev/null @@ -1,12 +0,0 @@ -SOURCES_METHOD=local -COMPILER=clang -COMPILER_INSTALL_METHOD=packages -COMPILER_PACKAGE_VERSION=6.0 -USE_LLVM_LIBRARIES_FROM_SYSTEM=0 -BUILD_METHOD=normal -BUILD_TARGETS=clickhouse -BUILD_TYPE=Debug -ENABLE_EMBEDDED_COMPILER=0 -CMAKE_FLAGS="-D CMAKE_C_FLAGS_ADD=-g0 -D CMAKE_CXX_FLAGS_ADD=-g0 -D ENABLE_TCMALLOC=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_UNWIND=0 -D ENABLE_ICU=0" - -# TODO it doesn't build with -D ENABLE_NETSSL=0 -D ENABLE_MONGODB=0 -D ENABLE_MYSQL=0 -D ENABLE_DATA_ODBC=0 diff --git a/ci/jobs/quick-build/run.sh b/ci/jobs/quick-build/run.sh index 0872b685e7c..744511e602b 100755 --- a/ci/jobs/quick-build/run.sh +++ b/ci/jobs/quick-build/run.sh @@ -12,6 +12,21 @@ cd "$(dirname $0)"/../.. . default-config +SOURCES_METHOD=local +COMPILER=clang +COMPILER_INSTALL_METHOD=packages +COMPILER_PACKAGE_VERSION=6.0 +USE_LLVM_LIBRARIES_FROM_SYSTEM=0 +BUILD_METHOD=normal +BUILD_TARGETS=clickhouse +BUILD_TYPE=Debug +ENABLE_EMBEDDED_COMPILER=0 + +CMAKE_FLAGS="-D CMAKE_C_FLAGS_ADD=-g0 -D CMAKE_CXX_FLAGS_ADD=-g0 -D ENABLE_TCMALLOC=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_UNWIND=0 -D ENABLE_ICU=0" +# TODO it doesn't build with -D ENABLE_NETSSL=0 -D ENABLE_MONGODB=0 -D ENABLE_MYSQL=0 -D ENABLE_DATA_ODBC=0 + +[[ $(uname) == "FreeBSD" ]] && COMPILER_PACKAGE_VERSION=devel + . get-sources.sh . prepare-toolchain.sh . install-libraries.sh diff --git a/ci/prepare-docker-image-ubuntu.sh b/ci/prepare-docker-image-ubuntu.sh index 1b3d3bd18f6..2880d7fc1e6 100755 --- a/ci/prepare-docker-image-ubuntu.sh +++ b/ci/prepare-docker-image-ubuntu.sh @@ -6,7 +6,7 @@ source default-config ./check-docker.sh # http://fl47l1n3.net/2015/12/24/binfmt/ -$SUDO apt-get -y install qemu-user-static +./install-os-packages.sh qemu-user-static pushd docker-multiarch diff --git a/ci/prepare-toolchain.sh b/ci/prepare-toolchain.sh index f90cb4fca4d..4718a854860 100755 --- a/ci/prepare-toolchain.sh +++ b/ci/prepare-toolchain.sh @@ -3,11 +3,10 @@ set -e -x source default-config -# TODO Non debian systems -apt-cache search cmake3 | grep -P '^cmake3 ' && $SUDO apt-get -y install cmake3 || $SUDO apt-get -y install cmake +./install-os-packages.sh cmake if [[ "$COMPILER_INSTALL_METHOD" == "packages" ]]; then - . install-compiler-from-packages.sh; + . install-compiler-from-packages.sh elif [[ "$COMPILER_INSTALL_METHOD" == "sources" ]]; then . install-compiler-from-sources.sh else diff --git a/ci/prepare-vagrant-image-freebsd.sh b/ci/prepare-vagrant-image-freebsd.sh index 81d021ca31f..6b5cfdd619c 100755 --- a/ci/prepare-vagrant-image-freebsd.sh +++ b/ci/prepare-vagrant-image-freebsd.sh @@ -3,7 +3,7 @@ set -e -x source default-config -$SUDO apt-get -y install vagrant virtualbox +./install-os-packages.sh vagrant-virtualbox pushd "vagrant-freebsd" vagrant up diff --git a/ci/run-with-vagrant.sh b/ci/run-with-vagrant.sh new file mode 100755 index 00000000000..67275c248d0 --- /dev/null +++ b/ci/run-with-vagrant.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -e -x + +[[ -r "vagrant-${1}/vagrant-ssh" ]] || die "Run prepare-vagrant-image-... first." + +pushd vagrant-$1 + +shopt -s extglob +vagrant ssh -c "mkdir ClickHouse" +scp -F vagrant-ssh -r ../../../ClickHouse/!(*build*) default:~/ClickHouse +vagrant ssh -c "cd ClickHouse/ci; $2" + +popd