Compile and doc fixes (#2410)

* Try fix travis

* fix

* Fix clickhouse-local shared-split link

* fix

* fix

* fix

* Build fixes

* Fix tinfo

* fix

* tinfo -> termcap

* termcap fix

* Better llvm version detect

* fix

* Debian postinst fix
Tests: use /usr/local if exists

* Compile and doc fixes

* fix

* fix

* fix

* macos fixes

* fix

* wip

* Update demangle.h

* Update demangle.cpp

CMakeLists.txt

@@ -128,14 +128,14 @@ endif ()
 #endif ()
 
 if (CMAKE_VERSION VERSION_LESS "3.8.0")
+    if (NOT MSVC)
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1z")
+    endif ()
 else ()
     set (CMAKE_CXX_STANDARD 17)
     set (CMAKE_CXX_EXTENSIONS 0) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS
     set (CMAKE_CXX_STANDARD_REQUIRED ON)
     set (CXX_FLAGS_INTERNAL_COMPILER "-std=c++1z")
-    # This needs to propagate to vendored projects in contrib
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
 endif ()
 
 set (CMAKE_BUILD_COLOR_MAKEFILE          ON)

contrib/CMakeLists.txt

@@ -1,5 +1,5 @@
 if (NOT MSVC)
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast")
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -std=c++1z")
 endif ()
 
 if (USE_INTERNAL_BOOST_LIBRARY)

contrib/libcityhash/src/config.h

@@ -116,3 +116,10 @@
 /* Define to the type of an unsigned integer type of width exactly 8 bits if
    such a type exists and the standard includes do not define it. */
 /* #undef uint8_t */
+
+#ifdef _MSC_VER
+    #include <basetsd.h>
+    typedef SSIZE_T ssize_t;
+#else
+    #include <sys/types.h>
+#endif

copy_headers.sh

@@ -17,7 +17,7 @@
 
 SOURCE_PATH=${1:-.}
 DST=${2:-$SOURCE_PATH/../headers}
-BUILD_PATH=${3:-$SOURCE_PATH/build}
+BUILD_PATH=${BUILD_PATH=${3:-$SOURCE_PATH/build}}
 
 PATH="/usr/local/bin:/usr/local/sbin:/usr/bin:$PATH"
 
@@ -30,24 +30,29 @@ START_HEADERS=$(echo \
     $SOURCE_PATH/dbms/src/Interpreters/SpecializedAggregator.h \
     $SOURCE_PATH/dbms/src/AggregateFunctions/AggregateFunction*.h)
 
+for header in $START_HEADERS; do
+    START_HEADERS_INCLUDE+="-include $header "
+done
+
 # Опция -mcx16 для того, чтобы выбиралось больше заголовочных файлов (с запасом).
 # The latter options are the same that are added while building packages.
 
+# TODO: Does not work on macos:
 GCC_ROOT=`$CLANG -v 2>&1 | grep "Selected GCC installation"| sed -n -e 's/^.*: //p'`
 
 for src_file in $(echo | $CLANG -M -xc++ -std=c++1z -Wall -Werror -msse4 -mcx16 -mpopcnt -O3 -g -fPIC -fstack-protector -D_FORTIFY_SOURCE=2 \
     -I $GCC_ROOT/include \
     -I $GCC_ROOT/include-fixed \
     $(cat "$BUILD_PATH/include_directories.txt") \
-    $(echo $START_HEADERS | sed -r -e 's/[^ ]+/-include \0/g') \
+    $START_HEADERS_INCLUDE \
     - |
     tr -d '\\' |
-    sed -r -e 's/^-\.o://');
+    sed -E -e 's/^-\.o://');
 do
     dst_file=$src_file;
-    [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -r -e "s!^$DESTDIR!!")
+    [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!")
-    dst_file=$(echo $dst_file | sed -r -e 's/build\///')    # for simplicity reasons, will put generated headers near the rest.
+    dst_file=$(echo $dst_file | sed -E -e 's/build\///')    # for simplicity reasons, will put generated headers near the rest.
-    mkdir -p "$DST/$(echo $dst_file | sed -r -e 's/\/[^/]*$/\//')";
+    mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')";
     cp "$src_file" "$DST/$dst_file";
 done
 
@@ -58,8 +63,8 @@ done
 for src_file in $(ls -1 $($CLANG -v -xc++ - <<<'' 2>&1 | grep '^ /' | grep 'include' | grep -E '/lib/clang/|/include/clang/')/*.h | grep -vE 'arm|altivec|Intrin');
 do
     dst_file=$src_file;
-    [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -r -e "s!^$DESTDIR!!")
+    [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!")
-    mkdir -p "$DST/$(echo $dst_file | sed -r -e 's/\/[^/]*$/\//')";
+    mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')";
     cp "$src_file" "$DST/$dst_file";
 done
 
@@ -67,15 +72,15 @@ done
 for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/libs/smart_ptr/include/boost/smart_ptr/detail/*);
 do
     dst_file=$src_file;
-    [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -r -e "s!^$DESTDIR!!")
+    [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!")
-    mkdir -p "$DST/$(echo $dst_file | sed -r -e 's/\/[^/]*$/\//')";
+    mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')";
     cp "$src_file" "$DST/$dst_file";
 done
 
 for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/boost/smart_ptr/detail/*);
 do
     dst_file=$src_file;
-    [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -r -e "s!^$DESTDIR!!")
+    [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!")
-    mkdir -p "$DST/$(echo $dst_file | sed -r -e 's/\/[^/]*$/\//')";
+    mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')";
     cp "$src_file" "$DST/$dst_file";
 done

dbms/CMakeLists.txt

@@ -246,8 +246,9 @@ target_include_directories (clickhouse_common_io BEFORE PUBLIC ${DOUBLE_CONVERSI
 target_include_directories (clickhouse_common_io BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
 
 if (USE_EMBEDDED_COMPILER)
-    add_custom_target(copy-headers ALL env CLANG=${CMAKE_CURRENT_BINARY_DIR}/src/Server/clickhouse-clang DESTDIR=${ClickHouse_SOURCE_DIR} ${ClickHouse_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/headers DEPENDS clickhouse-clang WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES ${ClickHouse_SOURCE_DIR}/copy_headers.sh)
+    add_custom_target(copy-headers ALL env CLANG=${CMAKE_CURRENT_BINARY_DIR}/src/Server/clickhouse-clang BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} bash -x ${ClickHouse_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/headers DEPENDS clickhouse-clang WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES ${ClickHouse_SOURCE_DIR}/copy_headers.sh)
     install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/headers DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse COMPONENT clickhouse)
+    add_dependencies(clickhouse-bundle copy-headers)
 endif ()
 
 add_subdirectory (tests)

dbms/tests/clickhouse-test-server

@@ -21,8 +21,8 @@ CONFIG_CLIENT=${CONFIG_CLIENT:=${CONFIG_CLIENT_DIR}client-test.xml}
 export CLICKHOUSE_CONFIG=${CLICKHOUSE_CONFIG:=${CONFIG_SERVER_DIR}server-test.xml}
 [ -x "$CUR_DIR/clickhouse-test" ] && TEST_DIR=${TEST_DIR=$CUR_DIR/}
 [ -d "$CUR_DIR/queries" ] && QUERIES_DIR=${QUERIES_DIR=$CUR_DIR/queries}
-[ ! -d "$QUERIES_DIR" ] && QUERIES_DIR=${QUERIES_DIR=/usr/local/share/clickhouse-test/queries}
+[ ! -d "$QUERIES_DIR" ] && [ -d "/usr/local/share/clickhouse-test/queries" ] && QUERIES_DIR=${QUERIES_DIR=/usr/local/share/clickhouse-test/queries}
-[ ! -d "$QUERIES_DIR" ] && QUERIES_DIR=${QUERIES_DIR=/usr/share/clickhouse-test/queries}
+[ ! -d "$QUERIES_DIR" ] && [ -d "/usr/share/clickhouse-test/queries" ] && QUERIES_DIR=${QUERIES_DIR=/usr/share/clickhouse-test/queries}
 CLICKHOUSE_EXTRACT_CONFIG=${CLICKHOUSE_EXTRACT_CONFIG:="${BIN_DIR}${CLICKHOUSE_BINARY}-extract-from-config --config=$CLICKHOUSE_CONFIG"}
 
 rm -rf $DATA_DIR

dbms/tests/integration/README.md

@@ -16,6 +16,8 @@ Don't use Docker from your system repository.
 * [py.test](https://docs.pytest.org/) testing framework. To install: `sudo -H pip install pytest`
 * [docker-compose](https://docs.docker.com/compose/) and additional python libraries. To install: `sudo -H pip install docker-compose docker dicttoxml kazoo PyMySQL`
 
+If you want install on modern debian/ubuntu: `sudo apt install -y docker docker-compose python-pytest python-dicttoxml python-docker python-pymysql python-kazoo`
+
 If you want to run the tests under a non-privileged user, you must add this user to `docker` group: `sudo usermod -aG docker $USER` and re-login.
 (You must close all your sessions (for example, restart your computer))
 To check, that you have access to Docker, run `docker ps`.

docs/en/development/build_osx.md

@@ -12,7 +12,7 @@ With appropriate changes, it should also work on any other Linux distribution.
 ## Install required compilers, tools, and libraries
 
 ```bash
-brew install cmake gcc icu4c mysql openssl unixodbc libtool gettext zlib readline boost --cc=gcc-7
+brew install cmake gcc icu4c mysql openssl unixodbc libtool gettext readline
 ```
 
 ## Checkout ClickHouse sources
@@ -34,7 +34,7 @@ For the latest release candidate, switch to the `testing` branch.
 ```bash
 mkdir build
 cd build
-cmake .. -DCMAKE_CXX_COMPILER=`which g++-7` -DCMAKE_C_COMPILER=`which gcc-7`
+cmake .. -DCMAKE_CXX_COMPILER=`which g++-8` -DCMAKE_C_COMPILER=`which gcc-8`
 make -j `sysctl -n hw.ncpu`
 cd ..
 ```

libs/libcommon/include/common/demangle.h

@@ -5,6 +5,7 @@
 
 /** Demangles C++ symbol name.
   * When demangling fails, returns the original name and sets status to non-zero.
+  * TODO: Write msvc version (now returns the same string)
   */
 std::string demangle(const char * name, int & status);
 

libs/libcommon/src/demangle.cpp

@@ -1,7 +1,17 @@
 #include <common/demangle.h>
-#include <cxxabi.h>
-#include <stdlib.h>
 
+#if _MSC_VER
+
+std::string demangle(const char * name, int & status)
+{
+    status = 0;
+    return name;
+}
+
+#else
+
+#include <stdlib.h>
+#include <cxxabi.h>
 
 std::string demangle(const char * name, int & status)
 {
@@ -26,3 +36,5 @@ std::string demangle(const char * name, int & status)
 
     return res;
 }
+
+#endif

libs/libcommon/src/shift10.cpp

@@ -1,6 +1,7 @@
 #include <common/shift10.h>
 #include <common/likely.h>
 #include <limits>
+#include <port/ssize_t.h>
 
 
 template <typename T>

utils/build/build_macos.sh

@@ -12,7 +12,7 @@ fi
 
 ## Install required compilers, tools, libraries
 
-brew install cmake gcc icu4c mysql openssl unixodbc libtool gettext zlib readline boost --cc=gcc-7
+brew install cmake gcc icu4c mysql openssl unixodbc libtool gettext readline
 
 ## Checkout ClickHouse sources