Merge pull request #2916 from proller/fix3

Build fixes [WIP]

cmake/find_boost.cmake

@@ -26,9 +26,9 @@ endif ()
 
 if (NOT Boost_SYSTEM_LIBRARY)
     set (USE_INTERNAL_BOOST_LIBRARY 1)
-    set (Boost_PROGRAM_OPTIONS_LIBRARY boost_program_options_internal)
     set (Boost_SYSTEM_LIBRARY boost_system_internal)
-    set (Boost_FILESYSTEM_LIBRARY boost_filesystem_internal)
+    set (Boost_PROGRAM_OPTIONS_LIBRARY boost_program_options_internal)
+    set (Boost_FILESYSTEM_LIBRARY boost_filesystem_internal ${Boost_SYSTEM_LIBRARY})
 
     set (Boost_INCLUDE_DIRS)
 

cmake/find_zlib.cmake

@@ -1,6 +1,4 @@
-if (NOT OS_FREEBSD AND NOT APPLE)
+option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED})
-    option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED})
-endif ()
 
 if (NOT USE_INTERNAL_ZLIB_LIBRARY)
     find_package (ZLIB)

contrib/boost-cmake/CMakeLists.txt

@@ -16,7 +16,7 @@ if (NOT MSVC)
     add_definitions(-Wno-unused-variable -Wno-deprecated-declarations)
 endif ()
 
-add_library(boost_program_options_internal
+add_library(boost_program_options_internal ${SPLIT_SHARED}
 ${LIBRARY_DIR}/libs/program_options/src/cmdline.cpp
 ${LIBRARY_DIR}/libs/program_options/src/config_file.cpp
 ${LIBRARY_DIR}/libs/program_options/src/convert.cpp
@@ -29,7 +29,7 @@ ${LIBRARY_DIR}/libs/program_options/src/value_semantic.cpp
 ${LIBRARY_DIR}/libs/program_options/src/variables_map.cpp
 ${LIBRARY_DIR}/libs/program_options/src/winmain.cpp)
 
-add_library(boost_filesystem_internal
+add_library(boost_filesystem_internal ${SPLIT_SHARED}
 ${LIBRARY_DIR}/libs/filesystem/src/codecvt_error_category.cpp
 ${LIBRARY_DIR}/libs/filesystem/src/operations.cpp
 ${LIBRARY_DIR}/libs/filesystem/src/path.cpp
@@ -39,9 +39,11 @@ ${LIBRARY_DIR}/libs/filesystem/src/unique_path.cpp
 ${LIBRARY_DIR}/libs/filesystem/src/utf8_codecvt_facet.cpp
 ${LIBRARY_DIR}/libs/filesystem/src/windows_file_codecvt.cpp)
 
-add_library(boost_system_internal
+add_library(boost_system_internal ${SPLIT_SHARED}
 ${LIBRARY_DIR}/libs/system/src/error_code.cpp)
 
+target_link_libraries (boost_filesystem_internal PUBLIC boost_system_internal)
+
 target_include_directories (boost_program_options_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
 target_include_directories (boost_filesystem_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
 target_include_directories (boost_system_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})

contrib/cctz-cmake/CMakeLists.txt

@@ -1,6 +1,6 @@
 SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/cctz)
 
-add_library(cctz
+add_library(cctz ${SPLIT_SHARED}
     ${LIBRARY_DIR}/src/civil_time_detail.cc
     ${LIBRARY_DIR}/src/time_zone_fixed.cc
     ${LIBRARY_DIR}/src/time_zone_format.cc

contrib/librdkafka-cmake/CMakeLists.txt

@@ -54,7 +54,7 @@ ${RDKAFKA_SOURCE_DIR}/lz4hc.c
 ${RDKAFKA_SOURCE_DIR}/rdgz.c
 )
 
-add_library(rdkafka STATIC ${SRCS})
+add_library(rdkafka ${SPLIT_SHARED} ${SRCS})
 target_include_directories(rdkafka PRIVATE include)
 target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR})
 target_link_libraries(rdkafka PUBLIC ${ZLIB_LIBRARIES} ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})

contrib/unixodbc-cmake/CMakeLists.txt

@@ -23,7 +23,7 @@ ${ODBC_SOURCE_DIR}/libltdl/loaders/preopen.c
 ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/libltdl/libltdlcS.c
 )
 
-add_library(ltdl STATIC ${SRCS})
+add_library(ltdl ${SPLIT_SHARED} ${SRCS})
 
 target_include_directories(ltdl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/libltdl)
 target_include_directories(ltdl PUBLIC ${ODBC_SOURCE_DIR}/libltdl)
@@ -273,7 +273,7 @@ ${ODBC_SOURCE_DIR}/lst/lstSetFreeFunc.c
 ${ODBC_SOURCE_DIR}/lst/_lstVisible.c
 )
 
-add_library(unixodbc STATIC ${SRCS})
+add_library(unixodbc ${SPLIT_SHARED} ${SRCS})
 
 target_link_libraries(unixodbc ltdl)
 

contrib/zstd-cmake/CMakeLists.txt

@@ -125,6 +125,6 @@ IF (ZSTD_LEGACY_SUPPORT)
             ${LIBRARY_LEGACY_DIR}/zstd_v07.h)
 ENDIF (ZSTD_LEGACY_SUPPORT)
 
-ADD_LIBRARY(zstd ${Sources} ${Headers})
+ADD_LIBRARY(zstd ${SPLIT_SHARED} ${Sources} ${Headers})
 
 target_include_directories (zstd PUBLIC ${LIBRARY_DIR})

dbms/src/Common/Stopwatch.cpp

@@ -5,9 +5,10 @@ StopwatchRUsage::Timestamp StopwatchRUsage::Timestamp::current()
 {
     StopwatchRUsage::Timestamp res;
 
-    ::rusage rusage;
+    ::rusage rusage {};
+#if !defined(__APPLE__)
     ::getrusage(RUSAGE_THREAD, &rusage);
-
+#endif
     res.user_ns = rusage.ru_utime.tv_sec * 1000000000UL + rusage.ru_utime.tv_usec * 1000UL;
     res.sys_ns = rusage.ru_stime.tv_sec * 1000000000UL + rusage.ru_stime.tv_usec * 1000UL;
     return res;

dbms/src/Common/TaskStatsInfoGetter.h

@@ -4,12 +4,7 @@
 #include <Core/Types.h>
 #include <boost/noncopyable.hpp>
 
-#if defined(__linux__)
 struct taskstats;
-#else
-struct taskstats {};
-#endif
-
 
 namespace DB
 {

dbms/src/Common/ThreadProfileEvents.h

@@ -8,6 +8,8 @@
 
 #if defined(__linux__)
 #include <linux/taskstats.h>
+#else
+struct taskstats {};
 #endif
 
 
@@ -87,8 +89,10 @@ struct RUsageCounters
 
     static RUsageCounters current(UInt64 real_time_ = getCurrentTimeNanoseconds())
     {
-        ::rusage rusage;
+        ::rusage rusage {};
+#if !defined(__APPLE__)
         ::getrusage(RUSAGE_THREAD, &rusage);
+#endif
         return RUsageCounters(rusage, real_time_);
     }
 
@@ -154,7 +158,7 @@ struct TasksStatsCounters
 {
     ::taskstats stat;
 
-    static TasksStatsCounters current() { return {}; }
+    static TasksStatsCounters current();
     static void incrementProfileEvents(const TasksStatsCounters &, const TasksStatsCounters &, ProfileEvents::Counters &) {}
     static void updateProfileEvents(TasksStatsCounters &, ProfileEvents::Counters &) {}
 };

dbms/src/Common/hex.h

@@ -1,4 +1,5 @@
 #pragma once
+#include <string>
 
 
 /// Maps 0..15 to 0..9A..F or 0..9a..f correspondingly.

dbms/src/Common/setThreadName.cpp

@@ -39,13 +39,17 @@ std::string getThreadName()
 {
     std::string name(16, '\0');
 
-#if defined(__FreeBSD__) || defined(__APPLE__)
+#if defined(__APPLE__)
-    if (pthread_get_name_np(pthread_self(), name.data(), name.size());
+    if (pthread_getname_np(pthread_self(), name.data(), name.size()))
-        throw DB::Exception("Cannot get thread name with pthread_get_name_np()", DB::ErrorCodes::PTHREAD_ERROR);
+        throw DB::Exception("Cannot get thread name with pthread_getname_np()", DB::ErrorCodes::PTHREAD_ERROR);
+#elif defined(__FreeBSD__)
+// TODO: make test. freebsd will have this function soon https://freshbsd.org/commit/freebsd/r337983
+//    if (pthread_get_name_np(pthread_self(), name.data(), name.size()))
+//        throw DB::Exception("Cannot get thread name with pthread_get_name_np()", DB::ErrorCodes::PTHREAD_ERROR);
 #else
     if (0 != prctl(PR_GET_NAME, name.data(), 0, 0, 0))
-#endif
         DB::throwFromErrno("Cannot get thread name with prctl(PR_GET_NAME)");
+#endif
 
     name.resize(std::strlen(name.data()));
     return name;

dbms/src/Dictionaries/Embedded/GeodataProviders/Entries.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Dictionaries/Embedded/GeodataProviders/Types.h>
+#include <string>
 
 struct RegionEntry
 {

dbms/src/Dictionaries/MongoDBBlockInputStream.cpp

@@ -56,8 +56,8 @@ namespace
             case Poco::MongoDB::ElementTraits<Int32>::TypeId:
                 static_cast<ColumnVector<T> &>(column).getData().push_back(static_cast<const Poco::MongoDB::ConcreteElement<Int32> &>(value).value());
                 break;
-            case Poco::MongoDB::ElementTraits<Int64>::TypeId:
+            case Poco::MongoDB::ElementTraits<Poco::Int64>::TypeId:
-                static_cast<ColumnVector<T> &>(column).getData().push_back(static_cast<const Poco::MongoDB::ConcreteElement<Int64> &>(value).value());
+                static_cast<ColumnVector<T> &>(column).getData().push_back(static_cast<const Poco::MongoDB::ConcreteElement<Poco::Int64> &>(value).value());
                 break;
             case Poco::MongoDB::ElementTraits<Float64>::TypeId:
                 static_cast<ColumnVector<T> &>(column).getData().push_back(static_cast<const Poco::MongoDB::ConcreteElement<Float64> &>(value).value());

dbms/src/Functions/FunctionsHashing.h

@@ -72,7 +72,7 @@ struct HalfMD5Impl
         union
         {
             unsigned char char_data[16];
-            Poco::UInt64 uint64_data;
+            uint64_t uint64_data;
         } buf;
 
         MD5_CTX ctx;

dbms/src/IO/AIO.h

@@ -2,6 +2,8 @@
 
 #if defined(__linux__)
 
+#include <boost/noncopyable.hpp>
+
 /// https://stackoverflow.com/questions/20759750/resolving-redefinition-of-timespec-in-time-h
 #define timespec linux_timespec
 #define timeval linux_timeval

dbms/src/IO/LimitReadBuffer.h

@@ -13,14 +13,14 @@ class LimitReadBuffer : public ReadBuffer
 {
 private:
     ReadBuffer & in;
-    UInt64 limit;
+    size_t limit;
     bool throw_exception;
     std::string exception_message;
 
     bool nextImpl() override;
 
 public:
-    LimitReadBuffer(ReadBuffer & in, UInt64 limit, bool throw_exception, std::string exception_message = {});
+    LimitReadBuffer(ReadBuffer & in, size_t limit, bool throw_exception, std::string exception_message = {});
     ~LimitReadBuffer() override;
 };
 

dbms/src/Interpreters/tests/internal_iotop.cpp

@@ -6,7 +6,6 @@
 #include <Poco/File.h>
 #include <Common/Stopwatch.h>
 #include <IO/WriteBufferFromString.h>
-
 #include <linux/taskstats.h>
 #include <sys/time.h>
 #include <sys/resource.h>

debian/rules

@@ -24,6 +24,12 @@ ifndef ENABLE_TESTS
     CMAKE_FLAGS += -DENABLE_TESTS=0
 endif
 
+ifndef MAKE_TARGET
+    MAKE_TARGET = clickhouse-bundle
+endif
+
+CMAKE_FLAGS += -DENABLE_UTILS=0
+
 #DEB_CLANG ?= $(shell which clang-6.0 || which clang-5.0 || which clang-4.0 || which clang || which clang-3.9 || which clang-3.8)
 
 #DEB_CC ?= gcc-7
@@ -70,7 +76,7 @@ override_dh_auto_configure:
 
 override_dh_auto_build:
 	# Fix for ninja. Do not add -O.
-	cd $(BUILDDIR) && $(MAKE) -j$(THREADS_COUNT)
+	cd $(BUILDDIR) && $(MAKE) -j$(THREADS_COUNT) $(MAKE_TARGET)
 	#cd $(BUILDDIR) && cmake --build . -- -j$(THREADS_COUNT) # cmake return true on error
 
 override_dh_auto_test:

libs/libcommon/CMakeLists.txt

@@ -104,8 +104,8 @@ target_link_libraries (
     pocoext
     ${CITYHASH_LIBRARIES}
     ${CCTZ_LIBRARY}
-    ${Boost_SYSTEM_LIBRARY}
     ${Boost_FILESYSTEM_LIBRARY}
+    ${Boost_SYSTEM_LIBRARY}
     ${MALLOC_LIBRARIES}
     ${CMAKE_THREAD_LIBS_INIT}
     ${GLIBC_COMPATIBILITY_LIBRARIES}

libs/libcommon/include/common/DateLUTImpl.h

@@ -3,8 +3,8 @@
 #include <common/Types.h>
 #include <common/likely.h>
 #include <common/strong_typedef.h>
-
 #include <ctime>
+#include <string>
 
 #define DATE_LUT_MAX (0xFFFFFFFFU - 86400)
 #define DATE_LUT_MAX_DAY_NUM (0xFFFFFFFFU / 86400)

libs/libcommon/include/common/Types.h

@@ -1,16 +1,16 @@
 #pragma once
+#include <cstdint>
+#include <cstddef>
 
-#include <Poco/Types.h>
+using Int8 = int8_t;
+using Int16 = int16_t;
+using Int32 = int32_t;
+using Int64 = int64_t;
 
-using Int8 = Poco::Int8;
+using UInt8 = uint8_t;
-using Int16 = Poco::Int16;
+using UInt16 = uint16_t;
-using Int32 = Poco::Int32;
+using UInt32 = uint32_t;
-using Int64 = Poco::Int64;
+using UInt64 = uint64_t;
-
-using UInt8 = Poco::UInt8;
-using UInt16 = Poco::UInt16;
-using UInt32 = Poco::UInt32;
-using UInt64 = Poco::UInt64;
 
 
 /** This is not the best way to overcome an issue of different definitions

libs/libdaemon/include/daemon/OwnFormattingChannel.h

@@ -3,9 +3,7 @@
 #include <Poco/AutoPtr.h>
 #include <Poco/Channel.h>
 #include <Poco/FormattingChannel.h>
-
+#include "OwnPatternFormatter.h"
-
-class OwnPatternFormatter;
 
 
 namespace DB

libs/libmysqlxx/include/mysqlxx/Types.h

@@ -1,7 +1,7 @@
 #pragma once
 
+#include <cstdint>
 #include <string>
-#include <Poco/Types.h>
 
 struct st_mysql;
 using MYSQL = st_mysql;
@@ -19,10 +19,10 @@ using MYSQL_FIELD = st_mysql_field;
 namespace mysqlxx
 {
 
-using UInt64 = Poco::UInt64;
+using UInt64 = uint64_t;
-using Int64 = Poco::Int64;
+using Int64 = int64_t;
-using UInt32 = Poco::UInt32;
+using UInt32 = uint32_t;
-using Int32 = Poco::Int32;
+using Int32 = int32_t;
 
 using MYSQL_LENGTH = unsigned long;
 using MYSQL_LENGTHS = MYSQL_LENGTH *;

utils/CMakeLists.txt

@@ -3,16 +3,21 @@ if (NOT NO_WERROR)
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
 endif ()
 
-add_subdirectory (compressor)
+# Utils used in package
-add_subdirectory (iotest)
-add_subdirectory (corrector_utf8)
 add_subdirectory (config-processor)
-add_subdirectory (zookeeper-cli)
-add_subdirectory (zookeeper-dump-tree)
-add_subdirectory (zookeeper-remove-by-list)
-add_subdirectory (zookeeper-create-entry-to-download-part)
-add_subdirectory (wikistat-loader)
-add_subdirectory (fill-factor)
-add_subdirectory (check-marks)
-add_subdirectory (test-data-generator)
 add_subdirectory (report)
+
+# Not used in package
+if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS)
+    add_subdirectory (compressor)
+    add_subdirectory (iotest)
+    add_subdirectory (corrector_utf8)
+    add_subdirectory (zookeeper-cli)
+    add_subdirectory (zookeeper-dump-tree)
+    add_subdirectory (zookeeper-remove-by-list)
+    add_subdirectory (zookeeper-create-entry-to-download-part)
+    add_subdirectory (wikistat-loader)
+    add_subdirectory (fill-factor)
+    add_subdirectory (check-marks)
+    add_subdirectory (test-data-generator)
+endif ()

utils/build/build_macos.sh

@@ -12,7 +12,7 @@ fi
 
 ## Install required compilers, tools, libraries
 
-brew install cmake gcc icu4c mariadb-connector-c openssl unixodbc libtool gettext readline
+brew install cmake gcc icu4c mariadb-connector-c openssl unixodbc libtool gettext readline librdkafka
 
 ## Checkout ClickHouse sources
 

utils/check_include.sh

@@ -14,6 +14,7 @@ inc="-I. \
 -I./contrib/libmetrohash/src \
 -I./contrib/double-conversion \
 -I./contrib/cityhash102/include \
+-I./contrib/murmurhash/include \
 -I./contrib/zookeeper/src/c/include \
 -I./contrib/zookeeper/src/c/generated \
 -I./contrib/libtcmalloc/include \
@@ -49,5 +50,5 @@ if [ -z $1 ]; then
 else
     echo -n "$1    "
     echo -n `grep "#include" $1| wc -l` "    "
-    echo -e "#include <$1> \n int main() {return 0;}" | time --format "%e %M" ${CXX:=g++-7} -c -std=c++1z $inc -x c++ -
+    echo "#include <$1> \n int main() {return 0;}" | time --format "%e %M" ${CXX:=g++-7} -c -std=c++1z $inc -x c++ -
 fi

utils/travis/normal.sh

@@ -32,12 +32,14 @@ cmake $CUR_DIR/../.. -DCMAKE_CXX_COMPILER=`which $DEB_CXX $CXX` -DCMAKE_C_COMPIL
     `# Use all possible contrib libs from system` \
     -DUNBUNDLED=1 \
     `# Disable all features` \
-    -DENABLE_CAPNP=0 -DENABLE_RDKAFKA=0 -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_JEMALLOC=0 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0 -DUSE_INTERNAL_LLVM_LIBRARY=0 $CMAKE_FLAGS \
+    -DENABLE_CAPNP=0 -DENABLE_RDKAFKA=0 -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_JEMALLOC=0 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0 -DUSE_INTERNAL_LLVM_LIBRARY=0 $CMAKE_FLAGS
-    && ninja clickhouse-bundle \
+
-    `# Skip tests:` \
+ninja clickhouse-bundle
-    `# 00281 requires internal compiler` \
+
-    `# 00428 requires sudo (not all vms allow this)` \
+# Skip tests:
-    `# 00385 runs infinitly (TODO: fix it)` \
+# 00281 requires internal compiler
-    && ( [ ! ${TEST_RUN=1} ] || ( ( cd $CUR_DIR/../.. && env TEST_OPT="--skip long compile 00428 00385 $TEST_OPT" TEST_PORT_RANDOM= TEST_PERF= bash -x dbms/tests/clickhouse-test-server ) || ${TEST_TRUE=false} ) )
+# 00428 requires sudo (not all vms allow this)
+# 00385 runs infinitly (TODO: fix it)
+[ ! ${TEST_RUN=1} ] || ( ( cd $CUR_DIR/../.. && env TEST_OPT="--skip long compile 00428 00385 $TEST_OPT" TEST_PORT_RANDOM= TEST_PERF= TEST_SERVER_STARTUP_WAIT=10 bash -x dbms/tests/clickhouse-test-server ) || ${TEST_TRUE=false} )
 
 date