Build fixes

cmake/find_rdkafka.cmake

@@ -24,6 +24,9 @@ if (RDKAFKA_LIB AND RDKAFKA_INCLUDE_DIR)
     if (SASL2_LIBRARY)
        list (APPEND RDKAFKA_LIBRARY ${SASL2_LIBRARY})
     endif ()
+    if (LZ4_LIBRARY)
+       list (APPEND RDKAFKA_LIBRARY ${LZ4_LIBRARY})
+    endif ()
 elseif (NOT MISSING_INTERNAL_RDKAFKA_LIBRARY)
     set (USE_INTERNAL_RDKAFKA_LIBRARY 1)
     set (RDKAFKA_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src")

dbms/src/Common/StringUtils/StringUtils.h

@@ -3,6 +3,7 @@
 #include <string>
 #include <cstring>
 #include <cstddef>
+#include <type_traits>
 
 
 namespace detail

dbms/src/Server/Compiler-5.0.0/CMakeLists.txt

@@ -47,7 +47,9 @@ LLVMSupport
 #PollyISL
 #PollyPPCG
 
-libtinfo.a
-
 PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads
 )
+
+if (NOT APPLE)
+    target_link_libraries(clickhouse-compiler-lib PRIVATE libtinfo.a)
+endif()

dbms/src/Server/Compiler-5.0.2

@@ -0,0 +1 @@
+Compiler-5.0.0

dbms/src/Server/Compiler-6.0.0/CMakeLists.txt

@@ -47,7 +47,9 @@ ${REQUIRED_LLVM_LIBRARIES}
 #PollyISL
 #PollyPPCG
 
-libtinfo.a
-
 PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads
 )
+
+if (NOT APPLE)
+    target_link_libraries(clickhouse-compiler-lib PRIVATE libtinfo.a)
+endif()

dbms/src/Server/Compiler-6.0.1

@@ -0,0 +1 @@
+Compiler-6.0.0

debian/.pbuilderrc

@@ -55,13 +55,6 @@ DEBIAN_SUITES=($UNSTABLE_CODENAME $TESTING_CODENAME $STABLE_CODENAME $STABLE_BAC
 # List of Ubuntu suites. Update these when needed.
 UBUNTU_SUITES=("bionic" "artful" "zesty" "xenial" "trusty" "devel")
 
-# Mirrors to use. Update these to your preferred mirror.
-DEBIAN_MIRROR=${DEBIAN_MIRROR:=deb.debian.org}
-UBUNTU_MIRROR=${UBUNTU_MIRROR:=archive.ubuntu.com}
-
-#DEBIAN_MIRROR="mirror.yandex.ru"
-#UBUNTU_MIRROR="mirror.yandex.ru"
-
 # Set a default distribution if none is used. Note that you can set your own default (i.e. ${DIST:="unstable"}).
 HOST_DIST=`lsb_release --short --codename`
 : ${DIST:="$HOST_DIST"}
@@ -104,7 +97,7 @@ ALLOWUNTRUSTED=${SET_ALLOWUNTRUSTED:=${ALLOWUNTRUSTED}}
 if $(echo ${DEBIAN_SUITES[@]} | grep -q $DIST); then
     # Debian configuration
     OSNAME=debian
-    MIRRORSITE="http://$DEBIAN_MIRROR/$OSNAME/"
+    MIRRORSITE=${SET_MIRRORSITE="http://deb.debian.org/$OSNAME/"}
     COMPONENTS="main contrib non-free"
     if $(echo "$STABLE_CODENAME stable" | grep -q $DIST); then
         OTHERMIRROR="$OTHERMIRROR | deb $MIRRORSITE $STABLE_BACKPORTS_SUITE $COMPONENTS"
@@ -119,11 +112,16 @@ if $(echo ${DEBIAN_SUITES[@]} | grep -q $DIST); then
             DEBOOTSTRAPOPTS+=( '--keyring' '/usr/share/keyrings/debian-archive-keyring.gpg' )
             # DEBOOTSTRAPOPTS+=( '--keyring' '/usr/share/keyrings/debian-keyring.gpg' )
     esac
-
 elif $(echo ${UBUNTU_SUITES[@]} | grep -q $DIST); then
     # Ubuntu configuration
     OSNAME=ubuntu
-    MIRRORSITE="http://$UBUNTU_MIRROR/$OSNAME/"
+
+    if [[ "$ARCH" == "amd64" || "$ARCH" == "i386" ]]; then
+        MIRRORSITE=${SET_MIRRORSITE="http://archive.ubuntu.com/ubuntu/"}
+    else
+        MIRRORSITE=${SET_MIRRORSITE="http://ports.ubuntu.com/ubuntu-ports/"}
+    fi
+
     COMPONENTS="main restricted universe multiverse"
 
     OTHERMIRROR="$OTHERMIRROR | deb $MIRRORSITE $DIST-updates main restricted universe multiverse"
@@ -142,7 +140,7 @@ else
     exit 1
 fi
 
-echo "using $NAME $OSNAME $DIST $ARCH $LOGNAME"
+echo "using $NAME $OSNAME $DIST $ARCH $LOGNAME $MIRRORSITE"
 
 case "$DIST" in
     "trusty")
@@ -161,28 +159,26 @@ case "$DIST" in
         ;;
 esac
 
-case "$DIST" in
-    "bionic" )
+if [ "$ARCH" != arm64 ]; then
+   case "$DIST" in
+    "bionic" | "experimental" | "unstable" | "testing")
         EXTRAPACKAGES+=" liblld-6.0-dev libclang-6.0-dev liblld-6.0 "
-        export CMAKE_FLAGS="-DENABLE_EMBEDDED_COMPILER=1 -DLLVM_VERSION_POSTFIX=-6.0 $CMAKE_FLAGS"
+        export CMAKE_FLAGS="-DLLVM_VERSION_POSTFIX=-6.0 $CMAKE_FLAGS"
         ;;
-    "artful" | "experimental" | "unstable" | "testing" )
-        if [ "$ARCH" != arm64 ]; then
+    "artful" )
         EXTRAPACKAGES+=" liblld-5.0-dev libclang-5.0-dev liblld-5.0 "
-            export CMAKE_FLAGS="-DENABLE_EMBEDDED_COMPILER=1 $CMAKE_FLAGS"
-        fi
         ;;
-esac
+   esac
+else
+    export CMAKE_FLAGS="-DENABLE_EMBEDDED_COMPILER=0 $CMAKE_FLAGS"
+fi
+
 
 # bundled zookeepeer have broken asm
-case "$ARCH" in
-    "arm64")
-        EXTRAPACKAGES+=" libzookeeper-mt-dev "
-        ;;
-esac
+[[ "$ARCH" == "arm64" ]] && EXTRAPACKAGES+=" libzookeeper-mt-dev "
 
 # will test symbols
-EXTRAPACKAGES+=" gdb "
+#EXTRAPACKAGES+=" gdb "
 
 [[ $CCACHE_PREFIX == 'distcc' ]] && EXTRAPACKAGES+=" $CCACHE_PREFIX "
 

debian/control

@@ -4,7 +4,6 @@ Priority: optional
 Maintainer: Alexey Milovidov <milovidov@yandex-team.ru>
 Build-Depends: debhelper (>= 9),
                cmake3 | cmake,
-               ninja-build [amd64 i386],
                gcc-7, g++-7,
                libc6-dev,
                libmariadbclient-dev | default-libmysqlclient-dev | libmysqlclient-dev,

debian/rules

@@ -43,11 +43,13 @@ endif
 
 CMAKE_FLAGS += -DCMAKE_CXX_COMPILER=`which $(CXX)` -DCMAKE_C_COMPILER=`which $(CC)`
 
-NINJA=$(shell which ninja)
+ifndef DISABLE_NINJA
+    NINJA=$(shell which ninja)
 ifneq ($(NINJA),)
         CMAKE_FLAGS += -GNinja
         export MAKE=$(NINJA)
 endif
+endif
 
 ifndef DH_VERBOSE
     CMAKE_FLAGS += -DCMAKE_VERBOSE_MAKEFILE=0

docs/ru/development/build_osx.md

@@ -12,7 +12,7 @@ With appropriate changes, build should work on any other OS X distribution.
 ## Install required compilers, tools, libraries
 
 ```bash
-brew install cmake gcc icu4c mysql openssl unixodbc libtool gettext homebrew/dupes/zlib readline boost --cc=gcc-7
+brew install cmake gcc icu4c mysql openssl unixodbc libtool gettext zlib readline boost --cc=gcc-7
 ```
 
 ## Checkout ClickHouse sources

release

@@ -1,5 +1,13 @@
 #!/bin/bash
 
+# Advanced usage:
+# Test gcc-8:
+# env DIST=bionic EXTRAPACKAGES="gcc-8 g++-8" DEB_CC=gcc-8 DEB_CXX=g++-8 CMAKE_FLAGS=" -DNO_WERROR=1 " ./release
+# Clang6 build:
+# env DIST=bionic EXTRAPACKAGES="clang-6.0 libstdc++-8-dev lld-6.0 liblld-6.0-dev libclang-6.0-dev liblld-6.0" DEB_CC=clang-6.0 DEB_CXX=clang++-6.0 CMAKE_FLAGS=" -DLLVM_VERSION_POSTFIX=-6.0 -DNO_WERROR=1 " ./release
+# Clang6 without internal compiler (for low memory arm64):
+# env DIST=bionic DISABLE_PARALLEL=1 EXTRAPACKAGES="clang-6.0 libstdc++-8-dev" DEB_CC=clang-6.0 DEB_CXX=clang++-6.0 CMAKE_FLAGS=" -DNO_WERROR=1 " ./release
+
 set -e
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
@@ -60,8 +68,8 @@ then
     # todo: нужно ли отключить libtcmalloc?
     LIBTCMALLOC_OPTS="-DENABLE_TCMALLOC=0"
     # GLIBC_COMPATIBILITY отключен по умолчанию
-    export DEB_CC=clang-5.0
-    export DEB_CXX=clang++-5.0
+    export DEB_CC=${DEB_CC=clang-5.0}
+    export DEB_CXX=${DEB_CXX=clang++-5.0}
     EXTRAPACKAGES="$EXTRAPACKAGES clang-5.0 lld-5.0"
 elif [[ $BUILD_TYPE == 'valgrind' ]]; then
     LIBTCMALLOC_OPTS="-DENABLE_TCMALLOC=0"

utils/build/build_debian_unbundled.sh

@@ -3,8 +3,7 @@
 ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)
 
 # also possible: DIST=bionic DIST=testing
-DIST=${DIST="unstable"}
-export DIST
+export DIST=${DIST=unstable}
 
 cd $ROOT_DIR
 . $ROOT_DIR/debian/.pbuilderrc

utils/build/build_freebsd.sh

@@ -36,11 +36,11 @@ cd ClickHouse/build
 cmake .. -DUNBUNDLED=1 -DUSE_STATIC_LIBRARIES=0
 # build with boost 1.64 from ports temporary broken
 
-make -C dbms/src/Server -j $(nproc || sysctl -n hw.ncpu || echo 2)
+make clickhouse-bundle -j $(nproc || sysctl -n hw.ncpu || echo 2)
 cd ../..
 
 #  Run server:
-# ClickHouse/build/dbms/src/Server/clickhouse --server --config-file=ClickHouse/dbms/src/Server/config.xml &
+# ClickHouse/build/dbms/src/Server/clickhouse-server --config-file=ClickHouse/dbms/src/Server/config.xml &
 
 #  Run client:
-# ClickHouse/build/dbms/src/Server/clickhouse --client
+# ClickHouse/build/dbms/src/Server/clickhouse-client

utils/build/build_macos.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+# How to build ClickHouse on Mac OS X
+#  Build should work on Mac OS X 10.12. If you're using earlier version, you can try to build ClickHouse using Gentoo Prefix and clang sl in this instruction.
+#  With appropriate changes, build should work on any other OS X distribution.
+
+## Install Homebrew
+
+if [ -z `which brew` ]; then
+    /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+fi
+
+## Install required compilers, tools, libraries
+
+brew install cmake gcc icu4c mysql openssl unixodbc libtool gettext zlib readline boost --cc=gcc-7
+
+## Checkout ClickHouse sources
+
+#  To get the latest stable version:
+
+git clone -b stable --recursive --depth=10 https://github.com/yandex/ClickHouse.git
+
+cd ClickHouse
+
+#  For development, switch to the `master` branch.
+#  For the latest release candidate, switch to the `testing` branch.
+
+## Build ClickHouse
+
+mkdir build
+cd build
+cmake .. -DCMAKE_CXX_COMPILER=`which g++-8 g++-7` -DCMAKE_C_COMPILER=`which gcc-8 gcc-7`
+make -j `sysctl -n hw.ncpu`
+
+cd ../..
+
+#  Run server:
+# ClickHouse/build/dbms/src/Server/clickhouse-server --config-file=ClickHouse/dbms/src/Server/config.xml &
+
+#  Run client:
+# ClickHouse/build/dbms/src/Server/clickhouse-client
+
+
+## Caveats
+#  If you intend to run clickhouse-server, make sure to increase system's maxfiles variable. See [MacOS.md](https://github.com/yandex/ClickHouse/blob/master/MacOS.md) for more details.