Merge pull request #2916 from proller/fix3

Build fixes [WIP]
2024-11-24 00:22:29 +00:00 · 2018-08-24 01:43:59 +03:00 · 2018-08-24 01:43:59 +03:00 · 0a8c3aeb89
commit 0a8c3aeb89
parent 08f6305dba ea80ca8984
28 changed files with 92 additions and 73 deletions
--- a/cmake/find_boost.cmake
+++ b/cmake/find_boost.cmake
@ -26,9 +26,9 @@ endif ()

 if (NOT Boost_SYSTEM_LIBRARY)
    set (USE_INTERNAL_BOOST_LIBRARY 1)
-    set (Boost_PROGRAM_OPTIONS_LIBRARY boost_program_options_internal)
    set (Boost_SYSTEM_LIBRARY boost_system_internal)
-    set (Boost_FILESYSTEM_LIBRARY boost_filesystem_internal)
+    set (Boost_PROGRAM_OPTIONS_LIBRARY boost_program_options_internal)
+    set (Boost_FILESYSTEM_LIBRARY boost_filesystem_internal ${Boost_SYSTEM_LIBRARY})

    set (Boost_INCLUDE_DIRS)

--- a/cmake/find_zlib.cmake
+++ b/cmake/find_zlib.cmake
@ -1,6 +1,4 @@
-if (NOT OS_FREEBSD AND NOT APPLE)
-    option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED})
-endif ()
+option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED})

 if (NOT USE_INTERNAL_ZLIB_LIBRARY)
    find_package (ZLIB)
--- a/contrib/boost-cmake/CMakeLists.txt
+++ b/contrib/boost-cmake/CMakeLists.txt
@ -16,7 +16,7 @@ if (NOT MSVC)
    add_definitions(-Wno-unused-variable -Wno-deprecated-declarations)
 endif ()

-add_library(boost_program_options_internal
+add_library(boost_program_options_internal ${SPLIT_SHARED}
 ${LIBRARY_DIR}/libs/program_options/src/cmdline.cpp
 ${LIBRARY_DIR}/libs/program_options/src/config_file.cpp
 ${LIBRARY_DIR}/libs/program_options/src/convert.cpp
@ -29,7 +29,7 @@ ${LIBRARY_DIR}/libs/program_options/src/value_semantic.cpp
 ${LIBRARY_DIR}/libs/program_options/src/variables_map.cpp
 ${LIBRARY_DIR}/libs/program_options/src/winmain.cpp)

-add_library(boost_filesystem_internal
+add_library(boost_filesystem_internal ${SPLIT_SHARED}
 ${LIBRARY_DIR}/libs/filesystem/src/codecvt_error_category.cpp
 ${LIBRARY_DIR}/libs/filesystem/src/operations.cpp
 ${LIBRARY_DIR}/libs/filesystem/src/path.cpp
@ -39,9 +39,11 @@ ${LIBRARY_DIR}/libs/filesystem/src/unique_path.cpp
 ${LIBRARY_DIR}/libs/filesystem/src/utf8_codecvt_facet.cpp
 ${LIBRARY_DIR}/libs/filesystem/src/windows_file_codecvt.cpp)

-add_library(boost_system_internal
+add_library(boost_system_internal ${SPLIT_SHARED}
 ${LIBRARY_DIR}/libs/system/src/error_code.cpp)

+target_link_libraries (boost_filesystem_internal PUBLIC boost_system_internal)
+
 target_include_directories (boost_program_options_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
 target_include_directories (boost_filesystem_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
 target_include_directories (boost_system_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
--- a/contrib/cctz-cmake/CMakeLists.txt
+++ b/contrib/cctz-cmake/CMakeLists.txt
@ -1,6 +1,6 @@
 SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/cctz)

-add_library(cctz
+add_library(cctz ${SPLIT_SHARED}
    ${LIBRARY_DIR}/src/civil_time_detail.cc
    ${LIBRARY_DIR}/src/time_zone_fixed.cc
    ${LIBRARY_DIR}/src/time_zone_format.cc
--- a/contrib/librdkafka-cmake/CMakeLists.txt
+++ b/contrib/librdkafka-cmake/CMakeLists.txt
@ -54,7 +54,7 @@ ${RDKAFKA_SOURCE_DIR}/lz4hc.c
 ${RDKAFKA_SOURCE_DIR}/rdgz.c
 )

-add_library(rdkafka STATIC ${SRCS})
+add_library(rdkafka ${SPLIT_SHARED} ${SRCS})
 target_include_directories(rdkafka PRIVATE include)
 target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR})
 target_link_libraries(rdkafka PUBLIC ${ZLIB_LIBRARIES} ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
--- a/contrib/unixodbc-cmake/CMakeLists.txt
+++ b/contrib/unixodbc-cmake/CMakeLists.txt
@ -23,7 +23,7 @@ ${ODBC_SOURCE_DIR}/libltdl/loaders/preopen.c
 ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/libltdl/libltdlcS.c
 )

-add_library(ltdl STATIC ${SRCS})
+add_library(ltdl ${SPLIT_SHARED} ${SRCS})

 target_include_directories(ltdl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/libltdl)
 target_include_directories(ltdl PUBLIC ${ODBC_SOURCE_DIR}/libltdl)
@ -273,7 +273,7 @@ ${ODBC_SOURCE_DIR}/lst/lstSetFreeFunc.c
 ${ODBC_SOURCE_DIR}/lst/_lstVisible.c
 )

-add_library(unixodbc STATIC ${SRCS})
+add_library(unixodbc ${SPLIT_SHARED} ${SRCS})

 target_link_libraries(unixodbc ltdl)

--- a/contrib/zstd-cmake/CMakeLists.txt
+++ b/contrib/zstd-cmake/CMakeLists.txt
@ -125,6 +125,6 @@ IF (ZSTD_LEGACY_SUPPORT)
            ${LIBRARY_LEGACY_DIR}/zstd_v07.h)
 ENDIF (ZSTD_LEGACY_SUPPORT)

-ADD_LIBRARY(zstd ${Sources} ${Headers})
+ADD_LIBRARY(zstd ${SPLIT_SHARED} ${Sources} ${Headers})

 target_include_directories (zstd PUBLIC ${LIBRARY_DIR})
--- a/dbms/src/Common/Stopwatch.cpp
+++ b/dbms/src/Common/Stopwatch.cpp
@ -5,9 +5,10 @@ StopwatchRUsage::Timestamp StopwatchRUsage::Timestamp::current()
 {
    StopwatchRUsage::Timestamp res;

-    ::rusage rusage;
+    ::rusage rusage {};
+#if !defined(__APPLE__)
    ::getrusage(RUSAGE_THREAD, &rusage);
-
+#endif
    res.user_ns = rusage.ru_utime.tv_sec * 1000000000UL + rusage.ru_utime.tv_usec * 1000UL;
    res.sys_ns = rusage.ru_stime.tv_sec * 1000000000UL + rusage.ru_stime.tv_usec * 1000UL;
    return res;
--- a/dbms/src/Common/TaskStatsInfoGetter.h
+++ b/dbms/src/Common/TaskStatsInfoGetter.h
@ -4,12 +4,7 @@
 #include <Core/Types.h>
 #include <boost/noncopyable.hpp>

-#if defined(__linux__)
 struct taskstats;
-#else
-struct taskstats {};
-#endif
-

 namespace DB
 {
--- a/dbms/src/Common/ThreadProfileEvents.h
+++ b/dbms/src/Common/ThreadProfileEvents.h
@ -8,6 +8,8 @@

 #if defined(__linux__)
 #include <linux/taskstats.h>
+#else
+struct taskstats {};
 #endif


@ -87,8 +89,10 @@ struct RUsageCounters

    static RUsageCounters current(UInt64 real_time_ = getCurrentTimeNanoseconds())
    {
-        ::rusage rusage;
+        ::rusage rusage {};
+#if !defined(__APPLE__)
        ::getrusage(RUSAGE_THREAD, &rusage);
+#endif
        return RUsageCounters(rusage, real_time_);
    }

@ -154,7 +158,7 @@ struct TasksStatsCounters
 {
    ::taskstats stat;

-    static TasksStatsCounters current() { return {}; }
+    static TasksStatsCounters current();
    static void incrementProfileEvents(const TasksStatsCounters &, const TasksStatsCounters &, ProfileEvents::Counters &) {}
    static void updateProfileEvents(TasksStatsCounters &, ProfileEvents::Counters &) {}
 };
--- a/dbms/src/Common/hex.h
+++ b/dbms/src/Common/hex.h
@ -1,4 +1,5 @@
 #pragma once
+#include <string>


 /// Maps 0..15 to 0..9A..F or 0..9a..f correspondingly.
--- a/dbms/src/Common/setThreadName.cpp
+++ b/dbms/src/Common/setThreadName.cpp
@ -39,13 +39,17 @@ std::string getThreadName()
 {
    std::string name(16, '\0');

-#if defined(__FreeBSD__) || defined(__APPLE__)
-    if (pthread_get_name_np(pthread_self(), name.data(), name.size());
-        throw DB::Exception("Cannot get thread name with pthread_get_name_np()", DB::ErrorCodes::PTHREAD_ERROR);
+#if defined(__APPLE__)
+    if (pthread_getname_np(pthread_self(), name.data(), name.size()))
+        throw DB::Exception("Cannot get thread name with pthread_getname_np()", DB::ErrorCodes::PTHREAD_ERROR);
+#elif defined(__FreeBSD__)
+// TODO: make test. freebsd will have this function soon https://freshbsd.org/commit/freebsd/r337983
+//    if (pthread_get_name_np(pthread_self(), name.data(), name.size()))
+//        throw DB::Exception("Cannot get thread name with pthread_get_name_np()", DB::ErrorCodes::PTHREAD_ERROR);
 #else
    if (0 != prctl(PR_GET_NAME, name.data(), 0, 0, 0))
-#endif
        DB::throwFromErrno("Cannot get thread name with prctl(PR_GET_NAME)");
+#endif

    name.resize(std::strlen(name.data()));
    return name;
--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/Entries.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/Entries.h
@ -1,6 +1,7 @@
 #pragma once

 #include <Dictionaries/Embedded/GeodataProviders/Types.h>
+#include <string>

 struct RegionEntry
 {
--- a/dbms/src/Dictionaries/MongoDBBlockInputStream.cpp
+++ b/dbms/src/Dictionaries/MongoDBBlockInputStream.cpp
@ -56,8 +56,8 @@ namespace
            case Poco::MongoDB::ElementTraits<Int32>::TypeId:
                static_cast<ColumnVector<T> &>(column).getData().push_back(static_cast<const Poco::MongoDB::ConcreteElement<Int32> &>(value).value());
                break;
-            case Poco::MongoDB::ElementTraits<Int64>::TypeId:
-                static_cast<ColumnVector<T> &>(column).getData().push_back(static_cast<const Poco::MongoDB::ConcreteElement<Int64> &>(value).value());
+            case Poco::MongoDB::ElementTraits<Poco::Int64>::TypeId:
+                static_cast<ColumnVector<T> &>(column).getData().push_back(static_cast<const Poco::MongoDB::ConcreteElement<Poco::Int64> &>(value).value());
                break;
            case Poco::MongoDB::ElementTraits<Float64>::TypeId:
                static_cast<ColumnVector<T> &>(column).getData().push_back(static_cast<const Poco::MongoDB::ConcreteElement<Float64> &>(value).value());
--- a/dbms/src/Functions/FunctionsHashing.h
+++ b/dbms/src/Functions/FunctionsHashing.h
@ -72,7 +72,7 @@ struct HalfMD5Impl
        union
        {
            unsigned char char_data[16];
-            Poco::UInt64 uint64_data;
+            uint64_t uint64_data;
        } buf;

        MD5_CTX ctx;
--- a/dbms/src/IO/AIO.h
+++ b/dbms/src/IO/AIO.h
@ -2,6 +2,8 @@

 #if defined(__linux__)

+#include <boost/noncopyable.hpp>
+
 /// https://stackoverflow.com/questions/20759750/resolving-redefinition-of-timespec-in-time-h
 #define timespec linux_timespec
 #define timeval linux_timeval
--- a/dbms/src/IO/LimitReadBuffer.h
+++ b/dbms/src/IO/LimitReadBuffer.h
@ -13,14 +13,14 @@ class LimitReadBuffer : public ReadBuffer
 {
 private:
    ReadBuffer & in;
-    UInt64 limit;
+    size_t limit;
    bool throw_exception;
    std::string exception_message;

    bool nextImpl() override;

 public:
-    LimitReadBuffer(ReadBuffer & in, UInt64 limit, bool throw_exception, std::string exception_message = {});
+    LimitReadBuffer(ReadBuffer & in, size_t limit, bool throw_exception, std::string exception_message = {});
    ~LimitReadBuffer() override;
 };

--- a/dbms/src/Interpreters/tests/internal_iotop.cpp
+++ b/dbms/src/Interpreters/tests/internal_iotop.cpp
@ -6,7 +6,6 @@
 #include <Poco/File.h>
 #include <Common/Stopwatch.h>
 #include <IO/WriteBufferFromString.h>
-
 #include <linux/taskstats.h>
 #include <sys/time.h>
 #include <sys/resource.h>
--- a/debian/rules
+++ b/debian/rules
@ -24,6 +24,12 @@ ifndef ENABLE_TESTS
    CMAKE_FLAGS += -DENABLE_TESTS=0
 endif

+ifndef MAKE_TARGET
+    MAKE_TARGET = clickhouse-bundle
+endif
+
+CMAKE_FLAGS += -DENABLE_UTILS=0
+
 #DEB_CLANG ?= $(shell which clang-6.0 || which clang-5.0 || which clang-4.0 || which clang || which clang-3.9 || which clang-3.8)

 #DEB_CC ?= gcc-7
@ -70,7 +76,7 @@ override_dh_auto_configure:

 override_dh_auto_build:
 	# Fix for ninja. Do not add -O.
-	cd $(BUILDDIR) && $(MAKE) -j$(THREADS_COUNT)
+	cd $(BUILDDIR) && $(MAKE) -j$(THREADS_COUNT) $(MAKE_TARGET)
 	#cd $(BUILDDIR) && cmake --build . -- -j$(THREADS_COUNT) # cmake return true on error

 override_dh_auto_test:
--- a/libs/libcommon/CMakeLists.txt
+++ b/libs/libcommon/CMakeLists.txt
@ -104,8 +104,8 @@ target_link_libraries (
    pocoext
    ${CITYHASH_LIBRARIES}
    ${CCTZ_LIBRARY}
-    ${Boost_SYSTEM_LIBRARY}
    ${Boost_FILESYSTEM_LIBRARY}
+    ${Boost_SYSTEM_LIBRARY}
    ${MALLOC_LIBRARIES}
    ${CMAKE_THREAD_LIBS_INIT}
    ${GLIBC_COMPATIBILITY_LIBRARIES}
--- a/libs/libcommon/include/common/DateLUTImpl.h
+++ b/libs/libcommon/include/common/DateLUTImpl.h
@ -3,8 +3,8 @@
 #include <common/Types.h>
 #include <common/likely.h>
 #include <common/strong_typedef.h>
-
 #include <ctime>
+#include <string>

 #define DATE_LUT_MAX (0xFFFFFFFFU - 86400)
 #define DATE_LUT_MAX_DAY_NUM (0xFFFFFFFFU / 86400)
--- a/libs/libcommon/include/common/Types.h
+++ b/libs/libcommon/include/common/Types.h
@ -1,16 +1,16 @@
 #pragma once
+#include <cstdint>
+#include <cstddef>

-#include <Poco/Types.h>
+using Int8 = int8_t;
+using Int16 = int16_t;
+using Int32 = int32_t;
+using Int64 = int64_t;

-using Int8 = Poco::Int8;
-using Int16 = Poco::Int16;
-using Int32 = Poco::Int32;
-using Int64 = Poco::Int64;
-
-using UInt8 = Poco::UInt8;
-using UInt16 = Poco::UInt16;
-using UInt32 = Poco::UInt32;
-using UInt64 = Poco::UInt64;
+using UInt8 = uint8_t;
+using UInt16 = uint16_t;
+using UInt32 = uint32_t;
+using UInt64 = uint64_t;


 /** This is not the best way to overcome an issue of different definitions
--- a/libs/libdaemon/include/daemon/OwnFormattingChannel.h
+++ b/libs/libdaemon/include/daemon/OwnFormattingChannel.h
@ -3,9 +3,7 @@
 #include <Poco/AutoPtr.h>
 #include <Poco/Channel.h>
 #include <Poco/FormattingChannel.h>
-
-
-class OwnPatternFormatter;
+#include "OwnPatternFormatter.h"


 namespace DB
--- a/libs/libmysqlxx/include/mysqlxx/Types.h
+++ b/libs/libmysqlxx/include/mysqlxx/Types.h
@ -1,7 +1,7 @@
 #pragma once

+#include <cstdint>
 #include <string>
-#include <Poco/Types.h>

 struct st_mysql;
 using MYSQL = st_mysql;
@ -19,10 +19,10 @@ using MYSQL_FIELD = st_mysql_field;
 namespace mysqlxx
 {

-using UInt64 = Poco::UInt64;
-using Int64 = Poco::Int64;
-using UInt32 = Poco::UInt32;
-using Int32 = Poco::Int32;
+using UInt64 = uint64_t;
+using Int64 = int64_t;
+using UInt32 = uint32_t;
+using Int32 = int32_t;

 using MYSQL_LENGTH = unsigned long;
 using MYSQL_LENGTHS = MYSQL_LENGTH *;
--- a/utils/CMakeLists.txt
+++ b/utils/CMakeLists.txt
@ -3,16 +3,21 @@ if (NOT NO_WERROR)
    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
 endif ()

-add_subdirectory (compressor)
-add_subdirectory (iotest)
-add_subdirectory (corrector_utf8)
+# Utils used in package
 add_subdirectory (config-processor)
-add_subdirectory (zookeeper-cli)
-add_subdirectory (zookeeper-dump-tree)
-add_subdirectory (zookeeper-remove-by-list)
-add_subdirectory (zookeeper-create-entry-to-download-part)
-add_subdirectory (wikistat-loader)
-add_subdirectory (fill-factor)
-add_subdirectory (check-marks)
-add_subdirectory (test-data-generator)
 add_subdirectory (report)
+
+# Not used in package
+if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS)
+    add_subdirectory (compressor)
+    add_subdirectory (iotest)
+    add_subdirectory (corrector_utf8)
+    add_subdirectory (zookeeper-cli)
+    add_subdirectory (zookeeper-dump-tree)
+    add_subdirectory (zookeeper-remove-by-list)
+    add_subdirectory (zookeeper-create-entry-to-download-part)
+    add_subdirectory (wikistat-loader)
+    add_subdirectory (fill-factor)
+    add_subdirectory (check-marks)
+    add_subdirectory (test-data-generator)
+endif ()
--- a/utils/build/build_macos.sh
+++ b/utils/build/build_macos.sh
@ -12,7 +12,7 @@ fi

 ## Install required compilers, tools, libraries

-brew install cmake gcc icu4c mariadb-connector-c openssl unixodbc libtool gettext readline
+brew install cmake gcc icu4c mariadb-connector-c openssl unixodbc libtool gettext readline librdkafka

 ## Checkout ClickHouse sources

--- a/utils/check_include.sh
+++ b/utils/check_include.sh
@ -14,6 +14,7 @@ inc="-I. \
 -I./contrib/libmetrohash/src \
 -I./contrib/double-conversion \
 -I./contrib/cityhash102/include \
+-I./contrib/murmurhash/include \
 -I./contrib/zookeeper/src/c/include \
 -I./contrib/zookeeper/src/c/generated \
 -I./contrib/libtcmalloc/include \
@ -49,5 +50,5 @@ if [ -z $1 ]; then
 else
    echo -n "$1    "
    echo -n `grep "#include" $1| wc -l` "    "
-    echo -e "#include <$1> \n int main() {return 0;}" | time --format "%e %M" ${CXX:=g++-7} -c -std=c++1z $inc -x c++ -
+    echo "#include <$1> \n int main() {return 0;}" | time --format "%e %M" ${CXX:=g++-7} -c -std=c++1z $inc -x c++ -
 fi
--- a/utils/travis/normal.sh
+++ b/utils/travis/normal.sh
@ -32,12 +32,14 @@ cmake $CUR_DIR/../.. -DCMAKE_CXX_COMPILER=`which $DEB_CXX $CXX` -DCMAKE_C_COMPIL
    `# Use all possible contrib libs from system` \
    -DUNBUNDLED=1 \
    `# Disable all features` \
-    -DENABLE_CAPNP=0 -DENABLE_RDKAFKA=0 -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_JEMALLOC=0 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0 -DUSE_INTERNAL_LLVM_LIBRARY=0 $CMAKE_FLAGS \
-    && ninja clickhouse-bundle \
-    `# Skip tests:` \
-    `# 00281 requires internal compiler` \
-    `# 00428 requires sudo (not all vms allow this)` \
-    `# 00385 runs infinitly (TODO: fix it)` \
-    && ( [ ! ${TEST_RUN=1} ] || ( ( cd $CUR_DIR/../.. && env TEST_OPT="--skip long compile 00428 00385 $TEST_OPT" TEST_PORT_RANDOM= TEST_PERF= bash -x dbms/tests/clickhouse-test-server ) || ${TEST_TRUE=false} ) )
+    -DENABLE_CAPNP=0 -DENABLE_RDKAFKA=0 -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_JEMALLOC=0 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0 -DUSE_INTERNAL_LLVM_LIBRARY=0 $CMAKE_FLAGS
+
+ninja clickhouse-bundle
+
+# Skip tests:
+# 00281 requires internal compiler
+# 00428 requires sudo (not all vms allow this)
+# 00385 runs infinitly (TODO: fix it)
+[ ! ${TEST_RUN=1} ] || ( ( cd $CUR_DIR/../.. && env TEST_OPT="--skip long compile 00428 00385 $TEST_OPT" TEST_PORT_RANDOM= TEST_PERF= TEST_SERVER_STARTUP_WAIT=10 bash -x dbms/tests/clickhouse-test-server ) || ${TEST_TRUE=false} )

 date