Build and test fixes:

Dont test symbols with gdb
    Travis: add clang6 + ubuntu bionic (disabled)
    Travis: dont run tests with gcc7 (too long, does not fit to limit)
    Allow use clang-7 as internal compiler: -DCMAKE_CXX_COMPILER=clang++-7 -DCMAKE_C_COMPILER=clang-7 -DLLVM_VERSION_POSTFIX=-7
    Debian: Fix package install (old version can't do service clickhouse disable_cron)
    Build fixes (ubuntu bionic + clang6.0)
    Update internal compiler clang6.0 to rc1

.travis.yml

@@ -16,7 +16,7 @@ matrix:
           packages: [ g++-7,            libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo ]
 
       env:
-        - MATRIX_EVAL="export CC=gcc-7 && export CXX=g++-7"
+        - MATRIX_EVAL="export CC=gcc-7 && export CXX=g++-7 && export TEST_RUN=false"
 
       script:
         - utils/travis/normal.sh
@@ -63,6 +63,27 @@ matrix:
         - utils/travis/pbuilder.sh
 
 
+# TODO: Can't bootstrap bionic on trusty host
+#    - os: linux
+#
+#      sudo: required
+#
+#      cache:
+#        timeout: 1000
+#        directories:
+#        - /var/cache/pbuilder/ccache
+#
+#      addons:
+#        apt:
+#          packages: [ pbuilder, fakeroot, debhelper ]
+#
+#      env:
+#        - MATRIX_EVAL="export CC=clang-6.0 && export CXX=clang++-6.0 && export DIST=bionic && export EXTRAPACKAGES=clang-6.0"
+#
+#      script:
+#        - utils/travis/pbuilder.sh
+
+
 # Cant fit to time limit (48min)
 #    - os: osx
 #      osx_image: xcode9.2

contrib/CMakeLists.txt

@@ -19,6 +19,7 @@ if (USE_INTERNAL_RE2_LIBRARY)
 endif ()
 
 if (USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY)
+    set (BUILD_TESTING ${ENABLE_TESTS} CACHE INTERNAL "")
     add_subdirectory (double-conversion)
 endif ()
 
@@ -81,15 +82,15 @@ if (NOT ARCH_ARM)
 endif ()
 
 if (USE_INTERNAL_RDKAFKA_LIBRARY)
-    set(RDKAFKA_BUILD_EXAMPLES OFF CACHE INTERNAL "")
-    set(RDKAFKA_BUILD_TESTS OFF CACHE INTERNAL "")
-    set(RDKAFKA_BUILD_STATIC ON CACHE INTERNAL "")
-    mark_as_advanced(ZLIB_INCLUDE_DIR)
+    set (RDKAFKA_BUILD_EXAMPLES OFF CACHE INTERNAL "")
+    set (RDKAFKA_BUILD_TESTS OFF CACHE INTERNAL "")
+    set (RDKAFKA_BUILD_STATIC ON CACHE INTERNAL "")
+    mark_as_advanced (ZLIB_INCLUDE_DIR)
     add_subdirectory (librdkafka)
 endif ()
 
 if (USE_INTERNAL_CAPNP_LIBRARY)
-    set(BUILD_TESTING 0 CACHE INTERNAL "")
+    set (BUILD_TESTING ${ENABLE_TESTS} CACHE INTERNAL "")
     set (_save ${CMAKE_CXX_EXTENSIONS})
     set (CMAKE_CXX_EXTENSIONS)
     add_subdirectory (capnproto/c++)

dbms/src/Server/Compiler-5.0.0/CMakeLists.txt

@@ -15,7 +15,7 @@ list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-lz")
 # Wrong library in freebsd:
 list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-l/usr/lib/libexecinfo.so")
 
-message(STATUS "Using libraries for LLVM: ${REQUIRED_LLVM_LIBRARIES}")
+message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
 
 target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS})
 

dbms/src/Server/Compiler-5.0.0/lld.cpp

@@ -41,6 +41,8 @@ LLVM_ATTRIBUTE_NORETURN static void die(const Twine &S) {
   exit(1);
 }
 
+#if 0
+
 static Flavor getFlavor(StringRef S) {
   return StringSwitch<Flavor>(S)
       .CasesLower("ld", "ld.lld", "gnu", Gnu)
@@ -90,6 +92,8 @@ static Flavor parseFlavor(std::vector<const char *> &V) {
   return parseProgname(Arg0);
 }
 
+#endif
+
 /// Universal linker main(). This linker emulates the gnu, darwin, or
 /// windows linker based on the argv[0] or -flavor option.
 int mainEntryClickHouseLLD(int Argc, char **Argv) {
@@ -99,12 +103,12 @@ int mainEntryClickHouseLLD(int Argc, char **Argv) {
   llvm_shutdown_obj Shutdown;
 
   std::vector<const char *> Args(Argv, Argv + Argc);
-/*
+#if 0
   switch (parseFlavor(Args)) {
   case Gnu:
-*/
+#endif
     return !elf::link(Args, true);
-/*
+#if 0
   case WinLink:
     return !coff::link(Args);
   case Darwin:
@@ -113,5 +117,5 @@ int mainEntryClickHouseLLD(int Argc, char **Argv) {
     die("lld is a generic driver.\n"
         "Invoke ld.lld (Unix), ld (macOS) or lld-link (Windows) instead.");
   }
-*/
+#endif
 }

dbms/src/Server/Compiler-6.0.0/CMakeLists.txt

@@ -15,7 +15,7 @@ list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-lz")
 # Wrong library in freebsd:
 list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-l/usr/lib/libexecinfo.so")
 
-message(STATUS "Using libraries for LLVM: ${REQUIRED_LLVM_LIBRARIES}")
+message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
 
 target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS})
 
@@ -39,6 +39,7 @@ lldReaderWriter
 lldYAML
 lldCommon
 lldCore
+#lldWasm
 
 ${REQUIRED_LLVM_LIBRARIES}
 

dbms/src/Server/Compiler-6.0.0/cc1as_main.cpp

@@ -397,7 +397,7 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
     if (Opts.ShowEncoding) {
       CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx);
       MCTargetOptions Options;
-      MAB = TheTarget->createMCAsmBackend(*MRI, Opts.Triple, Opts.CPU, Options);
+      MAB = TheTarget->createMCAsmBackend(*STI, *MRI, Options);
     }
     auto FOut = llvm::make_unique<formatted_raw_ostream>(*Out);
     Str.reset(TheTarget->createAsmStreamer(
@@ -415,8 +415,7 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
 
     MCCodeEmitter *CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx);
     MCTargetOptions Options;
-    MCAsmBackend *MAB = TheTarget->createMCAsmBackend(*MRI, Opts.Triple,
-                                                      Opts.CPU, Options);
+    MCAsmBackend *MAB = TheTarget->createMCAsmBackend(*STI, *MRI, Options);
     Triple T(Opts.Triple);
     Str.reset(TheTarget->createMCObjectStreamer(
                 T, Ctx, std::unique_ptr<MCAsmBackend>(MAB), *Out, std::unique_ptr<MCCodeEmitter>(CE), *STI,
@@ -442,7 +441,7 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
     auto Pair = StringRef(S).split('=');
     auto Sym = Pair.first;
     auto Val = Pair.second;
-    int64_t Value = 0;
+    int64_t Value;
     // We have already error checked this in the driver.
     Val.getAsInteger(0, Value);
     Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value);
@@ -468,7 +467,7 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
 }
 
 static void LLVMErrorHandler(void *UserData, const std::string &Message,
-                             bool /*GenCrashDiag*/) {
+                             bool GenCrashDiag) {
   DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
 
   Diags.Report(diag::err_fe_error_backend) << Message;
@@ -477,7 +476,7 @@ static void LLVMErrorHandler(void *UserData, const std::string &Message,
   exit(1);
 }
 
-int cc1as_main(ArrayRef<const char *> Argv, const char * /*Argv0*/, void * /*MainAddr*/) {
+int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
   // Initialize targets and assembly printers/parsers.
   InitializeAllTargetInfos();
   InitializeAllTargetMCs();

dbms/src/Server/Compiler-6.0.0/lld.cpp

@@ -29,6 +29,85 @@ using namespace lld;
 using namespace llvm;
 using namespace llvm::sys;
 
+enum Flavor {
+  Invalid,
+  Gnu,     // -flavor gnu
+  WinLink, // -flavor link
+  Darwin,  // -flavor darwin
+  Wasm,    // -flavor wasm
+};
+
+LLVM_ATTRIBUTE_NORETURN static void die(const Twine &S) {
+  errs() << S << "\n";
+  exit(1);
+}
+
+#if 0
+
+static Flavor getFlavor(StringRef S) {
+  return StringSwitch<Flavor>(S)
+      .CasesLower("ld", "ld.lld", "gnu", Gnu)
+      .CasesLower("wasm", "ld-wasm", Wasm)
+      .CaseLower("link", WinLink)
+      .CasesLower("ld64", "ld64.lld", "darwin", Darwin)
+      .Default(Invalid);
+}
+
+static bool isPETarget(const std::vector<const char *> &V) {
+  for (auto It = V.begin(); It + 1 != V.end(); ++It) {
+    if (StringRef(*It) != "-m")
+      continue;
+    StringRef S = *(It + 1);
+    return S == "i386pe" || S == "i386pep" || S == "thumb2pe" || S == "arm64pe";
+  }
+  return false;
+}
+
+static Flavor parseProgname(StringRef Progname) {
+#if __APPLE__
+  // Use Darwin driver for "ld" on Darwin.
+  if (Progname == "ld")
+    return Darwin;
+#endif
+
+#if LLVM_ON_UNIX
+  // Use GNU driver for "ld" on other Unix-like system.
+  if (Progname == "ld")
+    return Gnu;
+#endif
+
+  // Progname may be something like "lld-gnu". Parse it.
+  SmallVector<StringRef, 3> V;
+  Progname.split(V, "-");
+  for (StringRef S : V)
+    if (Flavor F = getFlavor(S))
+      return F;
+  return Invalid;
+}
+
+static Flavor parseFlavor(std::vector<const char *> &V) {
+  // Parse -flavor option.
+  if (V.size() > 1 && V[1] == StringRef("-flavor")) {
+    if (V.size() <= 2)
+      die("missing arg value for '-flavor'");
+    Flavor F = getFlavor(V[2]);
+    if (F == Invalid)
+      die("Unknown flavor: " + StringRef(V[2]));
+    V.erase(V.begin() + 1, V.begin() + 3);
+    return F;
+  }
+
+  // Deduct the flavor from argv[0].
+  StringRef Arg0 = path::filename(V[0]);
+  if (Arg0.endswith_lower(".exe"))
+    Arg0 = Arg0.drop_back(4);
+  return parseProgname(Arg0);
+}
+
+#endif
+
+/// Universal linker main(). This linker emulates the gnu, darwin, or
+/// windows linker based on the argv[0] or -flavor option.
 int mainEntryClickHouseLLD(int Argc, char **Argv) {
   // Standard set up, so program fails gracefully.
   sys::PrintStackTraceOnErrorSignal(Argv[0]);
@@ -36,5 +115,23 @@ int mainEntryClickHouseLLD(int Argc, char **Argv) {
   llvm_shutdown_obj Shutdown;
 
   std::vector<const char *> Args(Argv, Argv + Argc);
-  return !elf::link(Args, true);
+#if 0
+  switch (parseFlavor(Args)) {
+  case Gnu:
+    if (isPETarget(Args))
+      return !mingw::link(Args);
+#endif
+    return !elf::link(Args, true);
+#if 0
+  case WinLink:
+    return !coff::link(Args, true);
+  case Darwin:
+    return !mach_o::link(Args);
+  case Wasm:
+    return !wasm::link(Args, true);
+  default:
+    die("lld is a generic driver.\n"
+        "Invoke ld.lld (Unix), ld (macOS) or lld-link (Windows) instead.");
+  }
+#endif
 }

dbms/src/Server/Compiler-7.0.0

@@ -0,0 +1 @@
+Compiler-6.0.0

debian/.pbuilderrc

@@ -96,6 +96,10 @@ BUILDRESULT=${SET_BUILDRESULT}
 BUILDRESULT=${BUILDRESULT:="/var/cache/pbuilder/$NAME/result/"}
 APTCACHE="/var/cache/pbuilder/$NAME/aptcache/"
 BUILDPLACE="/var/cache/pbuilder/build/"
+ALLOWUNTRUSTED=${SET_ALLOWUNTRUSTED:=${ALLOWUNTRUSTED}}
+
+#DEBOOTSTRAPOPTS=( '--variant=buildd' $SET_DEBOOTSTRAPOPTS )
+
 
 if $(echo ${DEBIAN_SUITES[@]} | grep -q $DIST); then
     # Debian configuration
@@ -122,12 +126,12 @@ elif $(echo ${UBUNTU_SUITES[@]} | grep -q $DIST); then
     MIRRORSITE="http://$UBUNTU_MIRROR/$OSNAME/"
     COMPONENTS="main restricted universe multiverse"
 
-    OTHERMIRROR+="deb $MIRRORSITE $DIST-updates main restricted universe multiverse |"
-    OTHERMIRROR+="deb $MIRRORSITE $DIST-security main restricted universe multiverse |"
+    OTHERMIRROR="$OTHERMIRROR | deb $MIRRORSITE $DIST-updates main restricted universe multiverse"
+    OTHERMIRROR="$OTHERMIRROR | deb $MIRRORSITE $DIST-security main restricted universe multiverse"
 
     case "$DIST" in
        "trusty" | "xenial" )
-           OTHERMIRROR+="deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/$OSNAME $DIST main |"
+           OTHERMIRROR="$OTHERMIRROR | deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/$OSNAME $DIST main"
            ALLOWUNTRUSTED=yes
            ;;
     esac
@@ -158,7 +162,11 @@ case "$DIST" in
 esac
 
 case "$DIST" in
-    "bionic" | "artful" | "experimental" | "unstable" | "testing" )
+    "bionic" )
+        EXTRAPACKAGES+=" liblld-6.0-dev libclang-6.0-dev liblld-6.0 "
+        export CMAKE_FLAGS="-DUSE_EMBEDDED_COMPILER=1 -DLLVM_VERSION_POSTFIX=-6.0 $CMAKE_FLAGS"
+        ;;
+    "artful" | "experimental" | "unstable" | "testing" )
         EXTRAPACKAGES+=" liblld-5.0-dev libclang-5.0-dev liblld-5.0 "
         export CMAKE_FLAGS="-DUSE_EMBEDDED_COMPILER=1 $CMAKE_FLAGS"
         ;;
@@ -174,6 +182,8 @@ esac
 # will test symbols
 EXTRAPACKAGES+=" gdb "
 
+[[ $CCACHE_PREFIX == 'distcc' ]] && EXTRAPACKAGES+=" $CCACHE_PREFIX "
+
 export DEB_BUILD_OPTIONS=parallel=`nproc`
 
 # Floating bug with permissions:
@@ -181,7 +191,6 @@ export DEB_BUILD_OPTIONS=parallel=`nproc`
 [ -n "$CCACHEDIR" ] && sudo chmod -R a+rwx $CCACHEDIR
 # chown -R $BUILDUSERID:$BUILDUSERID $CCACHEDIR
 
-# echo "DEBOOTSTRAPOPTS = ${DEBOOTSTRAPOPTS[@]}"
 
 # Use current dir to make package (by default should have src archive)
 # echo "3.0 (native)" > debian/source/format
@@ -191,3 +200,7 @@ export DEB_BUILD_OPTIONS=parallel=`nproc`
 DEBBUILDOPTS="--source-option=--format=\"3.0 (native)\""
 
 HOOKDIR="debian/pbuilder-hooks"
+
+#echo "DEBOOTSTRAPOPTS=${DEBOOTSTRAPOPTS[@]}"
+#echo "ALLOWUNTRUSTED=${ALLOWUNTRUSTED} OTHERMIRROR=${OTHERMIRROR}"
+#echo "EXTRAPACKAGES=${EXTRAPACKAGES}"

debian/clickhouse-server-base.preinst

@@ -2,5 +2,5 @@
 
 if [ "$1" = "upgrade" ]; then
     # Return etc/cron.d/clickhouse-server to original state
-    service clickhouse-server disable_cron
+    service clickhouse-server disable_cron || true
 fi

debian/clickhouse-server-base.prerm

@@ -2,5 +2,5 @@
 
 if [ "$1" = "upgrade" || [ "$1" = "remove" ]; then
     # Return etc/cron.d/clickhouse-server to original state
-    service clickhouse-server disable_cron
+    service clickhouse-server disable_cron || true
 fi

debian/pbuilder-hooks/B90test-server

@@ -35,4 +35,4 @@ TEST_TRUE=${TEST_TRUE:=true}
 service clickhouse-server stop
 
 # Test debug symbols
-gdb -ex quit --args /usr/bin/clickhouse-server
+# gdb -ex quit --args /usr/bin/clickhouse-server

debian/rules

@@ -19,7 +19,9 @@ DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH)
 #endif
 #DEB_BUILD_OPTIONS+=parallel=$(THREADS_COUNT)
 
-CMAKE_FLAGS += -DENABLE_TESTS=0
+ifndef ENABLE_TESTS
+    CMAKE_FLAGS += -DENABLE_TESTS=0
+endif
 
 #DEB_CLANG ?= $(shell which clang-6.0 || which clang-5.0 || which clang-4.0 || which clang || which clang-3.9 || which clang-3.8)
 

release

@@ -39,7 +39,7 @@ do
         shift
     elif [[ $1 == '--fast' ]]; then
         # Wrong but fast pbuilder mode: create base package with all depends
-        EXTRAPACKAGES+=" debhelper cmake gcc-7 g++-7 libc6-dev libmariadbclient-dev libicu-dev libltdl-dev libreadline-dev libssl-dev unixodbc-dev psmisc gdb bash expect python python-lxml python-termcolor curl perl sudo"
+        EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake gcc-7 g++-7 libc6-dev libmariadbclient-dev libicu-dev libltdl-dev libreadline-dev libssl-dev unixodbc-dev psmisc bash expect python python-lxml python-termcolor curl perl sudo"
         shift
     else
         echo "Unknown option $1"
@@ -58,7 +58,7 @@ then
     # GLIBC_COMPATIBILITY отключен по умолчанию
     export DEB_CC=clang-5.0
     export DEB_CXX=clang++-5.0
-    EXTRAPACKAGES+=" clang-5.0 "
+    EXTRAPACKAGES="$EXTRAPACKAGES clang-5.0 "
 elif [[ $BUILD_TYPE == 'valgrind' ]]; then
     LIBTCMALLOC_OPTS="-DENABLE_TCMALLOC=0"
     VERSION_POSTFIX+=-$BUILD_TYPE
@@ -90,9 +90,12 @@ else
     export DIST=${DIST:=artful}
     export SET_BUILDRESULT=${SET_BUILDRESULT:=$CURDIR/..}
     . $CURDIR/debian/.pbuilderrc
+
+
     if [[ -n "$FORCE_PBUILDER_CREATE" || ! -e "$BASETGZ" ]] ; then
         echo Creating base system $BASETGZ
-        sudo --preserve-env pbuilder create --configfile $CURDIR/debian/.pbuilderrc $PBUILDER_OPT
+        [ ! -e "/usr/share/debootstrap/scripts/${DIST}" ] && sudo ln -s gutsy /usr/share/debootstrap/scripts/${DIST}
+        sudo --preserve-env bash -x pbuilder create --configfile $CURDIR/debian/.pbuilderrc $PBUILDER_OPT
     fi
 
     # Update every 3 days (60*24*3 minutes)

utils/travis/normal.sh

@@ -8,6 +8,7 @@ set -e
 set -x
 
 TEST_TRUE=${TEST_TRUE:=false}
+TEST_RUN=${TEST_RUN:=true}
 
 # clean not used ~600mb
 [ -n "$TRAVIS" ] && rm -rf .git contrib/poco/openssl
@@ -31,4 +32,4 @@ cmake .. -DCMAKE_CXX_COMPILER=`which $CXX` -DCMAKE_C_COMPILER=`which $CC` \
     `# Skip tests:` \
     `# 00281 requires internal compiler` \
     `# 00428 requires sudo (not all vms allow this)` \
-    && ( ( cd .. && env TEST_OPT="--no-long --no-shard --skip 00281 00428 $TEST_OPT" bash -x dbms/tests/clickhouse-test-server ) || $TEST_TRUE )
+    && ( $TEST_RUN && ( ( cd .. && env TEST_OPT="--no-long --no-shard --skip 00281 00428 $TEST_OPT" bash -x dbms/tests/clickhouse-test-server ) || $TEST_TRUE ) || true )

utils/travis/pbuilder.sh

@@ -22,6 +22,5 @@ env TEST_RUN=1 \
     CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Debug -DUNBUNDLED=1 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DENABLE_CAPNP=0 -DENABLE_RDKAFKA=0 -DUSE_EMBEDDED_COMPILER=0 -DCMAKE_C_FLAGS_ADD='-O0 -g0' -DCMAKE_CXX_FLAGS_ADD='-O0 -g0' $CMAKE_FLAGS" \
     `# Use all possible contrib libs from system` \
     `# psmisc - killall` \
-    `# gdb - symbol test in pbuilder` \
-    EXTRAPACKAGES="psmisc gdb clang-5.0 libc++abi-dev libc++-dev libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libzookeeper-mt-dev libsparsehash-dev librdkafka-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev $EXTRAPACKAGES" \
+    EXTRAPACKAGES="psmisc clang-5.0 libc++abi-dev libc++-dev libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libzookeeper-mt-dev libsparsehash-dev librdkafka-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev $EXTRAPACKAGES" \
     ./release --pbuilder $RELEASE_OPT