Build fixes (#6016)

* Fix building without submodules

* Fix more gcc9 warnings

* was wrong!
../dbms/src/IO/WriteBufferAIO.cpp:277:54: error: result of comparison 'ssize_t' (aka 'long') > 9223372036854775807 is always false [-Werror,-Wtautological-type-limit-compare]
    if ((static_cast<ssize_t>(flush_buffer.offset()) > std::numeric_limits<off_t>::max()) ||
         ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

* mimalloc off MI_OVERRIDE

* Fix freebsd build

* Fix shared build

* Fix build in gcc9

* Fix split build

* fix

* fix

* fix

* fix

* clean

* zstd

* Update CMakeLists.txt

* Update Defines.h

* Try fix -Wsign-compare

* Freebsd fix

* Add missing libs

* Fix double-conversion include for copy-headers

* Fix zlib link

* Fix includes for arcadia

* Fix includes for arcadia

* Fix includes for arcadia

* Freebsd fix

* Arcadia fixes

* Update QueryProfiler.cpp

* Freebsd fix

* clean

* 19.11: Fixes for arcadia porting (#6223)

* fix glibc-compatibility

* M_LIBRARY fix

* wip

* Fix

* Fix arm build

* unwind fix

* Update CMakeLists.txt
This commit is contained in:
proller 2019-08-04 03:19:03 +03:00 committed by alexey-milovidov
parent 224bc4df97
commit 77bd2624ea
37 changed files with 88 additions and 63 deletions

View File

@ -262,10 +262,10 @@ if (USE_STATIC_LIBRARIES AND HAVE_NO_PIE)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAG_NO_PIE}")
endif ()
if (NOT SANITIZE)
if (NOT SANITIZE AND NOT SPLIT_SHARED_LIBRARIES)
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined")
endif()
endif ()
include (cmake/find_unwind.cmake)
@ -311,8 +311,11 @@ if (OS_LINUX AND NOT UNBUNDLED AND (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_L
# There are two variants of C++ library: libc++ (from LLVM compiler infrastructure) and libstdc++ (from GCC).
if (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING)
# TODO: Allow to use non-static library as well.
set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_static${${CMAKE_POSTFIX_VARIABLE}}.a")
if (USE_STATIC_LIBRARIES)
set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_static${${CMAKE_POSTFIX_VARIABLE}}.a")
else ()
set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_shared${${CMAKE_POSTFIX_VARIABLE}}.so")
endif ()
else ()
set (EXCEPTION_HANDLING_LIBRARY "-lgcc_eh")
endif ()
@ -352,6 +355,10 @@ if (OS_LINUX AND NOT UNBUNDLED AND (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_L
message(STATUS "Default libraries: ${DEFAULT_LIBS}")
endif ()
if (NOT GLIBC_COMPATIBILITY)
set (M_LIBRARY m)
endif ()
if (DEFAULT_LIBS)
# Add default libs to all targets as the last dependency.
set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})

View File

@ -129,7 +129,7 @@ find_package_handle_standard_args(ODBC
)
if(ODBC_FOUND)
set(ODBC_LIBRARIES ${ODBC_LIBRARY} ${_odbc_required_libs_paths})
set(ODBC_LIBRARIES ${ODBC_LIBRARY} ${_odbc_required_libs_paths} ${LTDL_LIBRARY})
set(ODBC_INCLUDE_DIRS ${ODBC_INCLUDE_DIR})
set(ODBC_DEFINITIONS ${PC_ODBC_CFLAGS_OTHER})
endif()

View File

@ -16,7 +16,12 @@ list(APPEND dirs ${dirs1})
get_property (dirs1 TARGET roaring PROPERTY INCLUDE_DIRECTORIES)
list(APPEND dirs ${dirs1})
if (USE_INTERNAL_BOOST_LIBRARY)
if (TARGET double-conversion)
get_property (dirs1 TARGET double-conversion PROPERTY INCLUDE_DIRECTORIES)
list(APPEND dirs ${dirs1})
endif ()
if (TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY})
get_property (dirs1 TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY} PROPERTY INCLUDE_DIRECTORIES)
list(APPEND dirs ${dirs1})
endif ()

View File

@ -44,6 +44,7 @@ set( thriftcpp_threads_SOURCES
add_library(${THRIFT_LIBRARY} ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES})
set_target_properties(${THRIFT_LIBRARY} PROPERTIES CXX_STANDARD 14) # REMOVE after https://github.com/apache/thrift/pull/1641
target_include_directories(${THRIFT_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src PRIVATE ${Boost_INCLUDE_DIRS})
target_link_libraries(${THRIFT_LIBRARY} PRIVATE Threads::Threads)

View File

@ -31,3 +31,7 @@ set(SRCS
add_library(brotli ${SRCS})
target_include_directories(brotli PUBLIC ${BROTLI_SOURCE_DIR}/include)
if(M_LIBRARY)
target_link_libraries(brotli PRIVATE ${M_LIBRARY})
endif()

View File

@ -10,5 +10,4 @@ ${LIBRARY_DIR}/double-conversion/fast-dtoa.cc
${LIBRARY_DIR}/double-conversion/fixed-dtoa.cc
${LIBRARY_DIR}/double-conversion/strtod.cc)
target_include_directories(double-conversion SYSTEM PUBLIC "${LIBRARY_DIR}")
target_include_directories(double-conversion SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}")

View File

@ -25,3 +25,6 @@ add_library(h3 ${SRCS})
target_include_directories(h3 SYSTEM PUBLIC ${H3_SOURCE_DIR}/include)
target_include_directories(h3 SYSTEM PUBLIC ${H3_BINARY_DIR}/include)
target_compile_definitions(h3 PRIVATE H3_HAVE_VLA)
if(M_LIBRARY)
target_link_libraries(h3 PRIVATE ${M_LIBRARY})
endif()

View File

@ -65,7 +65,7 @@ add_library(rdkafka ${SRCS})
target_include_directories(rdkafka SYSTEM PUBLIC include)
target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR}) # Because weird logic with "include_next" is used.
target_include_directories(rdkafka SYSTEM PRIVATE ${ZSTD_INCLUDE_DIR}/common) # Because wrong path to "zstd_errors.h" is used.
target_link_libraries(rdkafka PUBLIC ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${LIBGSASL_LIBRARY})
target_link_libraries(rdkafka PRIVATE ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${LIBGSASL_LIBRARY} Threads::Threads)
if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY)
target_link_libraries(rdkafka PUBLIC ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
target_link_libraries(rdkafka PRIVATE ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
endif()

View File

@ -26,6 +26,7 @@ set(LIBUNWIND_SOURCES
add_library(unwind_static ${LIBUNWIND_SOURCES})
target_include_directories(unwind_static PUBLIC ${LIBUNWIND_SOURCE_DIR}/include)
target_include_directories(unwind_static SYSTEM BEFORE PUBLIC ${LIBUNWIND_SOURCE_DIR}/include)
target_compile_definitions(unwind_static PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY)
target_compile_options(unwind_static PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all -nostdinc++ -fno-rtti)
target_link_libraries(unwind_static PRIVATE Threads::Threads ${CMAKE_DL_LIBS})

View File

@ -52,7 +52,10 @@ set(SRCS
)
add_library(libxml2 ${SRCS})
target_link_libraries(libxml2 ${ZLIB_LIBRARIES})
target_link_libraries(libxml2 PRIVATE ${ZLIB_LIBRARIES} ${CMAKE_DL_LIBS})
if(M_LIBRARY)
target_link_libraries(libxml2 PRIVATE ${M_LIBRARY})
endif()
target_include_directories(libxml2 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include)
target_include_directories(libxml2 PUBLIC ${LIBXML2_SOURCE_DIR}/include)

View File

@ -60,6 +60,11 @@ endif()
add_library(mysqlclient ${SRCS})
target_link_libraries(mysqlclient PRIVATE ${CMAKE_DL_LIBS} Threads::Threads)
if(M_LIBRARY)
target_link_libraries(mysqlclient PRIVATE ${M_LIBRARY})
endif()
if(OPENSSL_LIBRARIES)
target_link_libraries(mysqlclient PRIVATE ${OPENSSL_LIBRARIES})
target_compile_definitions(mysqlclient PRIVATE -D HAVE_OPENSSL -D HAVE_TLS)

View File

@ -32,6 +32,7 @@ target_include_directories(ltdl PUBLIC ${ODBC_SOURCE_DIR}/libltdl/libltdl)
target_compile_definitions(ltdl PRIVATE -DHAVE_CONFIG_H -DLTDL -DLTDLOPEN=libltdlc)
target_compile_options(ltdl PRIVATE -Wno-constant-logical-operand -Wno-unknown-warning-option -O2)
target_link_libraries(ltdl PRIVATE ${CMAKE_DL_LIBS})
set(SRCS

View File

@ -160,9 +160,6 @@ if (OS_FREEBSD)
endif ()
if (USE_UNWIND)
target_compile_definitions (clickhouse_common_io PRIVATE USE_UNWIND=1)
target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${UNWIND_INCLUDE_DIR})
if (NOT USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING)
target_link_libraries (clickhouse_common_io PRIVATE ${UNWIND_LIBRARY})
endif ()
@ -253,10 +250,6 @@ target_link_libraries(clickhouse_common_io
roaring
)
if(ZSTD_LIBRARY)
target_link_libraries(clickhouse_common_io PUBLIC ${ZSTD_LIBRARY})
endif()
if (USE_RDKAFKA)
target_link_libraries(dbms PRIVATE ${CPPKAFKA_LIBRARY} ${RDKAFKA_LIBRARY})
if(NOT USE_INTERNAL_RDKAFKA_LIBRARY)
@ -312,11 +305,14 @@ target_include_directories(dbms SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
if (NOT USE_INTERNAL_LZ4_LIBRARY)
target_include_directories(dbms SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR})
endif ()
if (ZSTD_LIBRARY)
target_link_libraries(dbms PRIVATE ${ZSTD_LIBRARY})
endif()
if (NOT USE_INTERNAL_ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR)
target_include_directories(dbms SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
endif ()
if (NOT USE_INTERNAL_BOOST_LIBRARY)
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
endif ()

View File

@ -5,7 +5,9 @@
#include <vector>
#include <boost/noncopyable.hpp>
#include <common/likely.h>
#include <sanitizer/asan_interface.h>
#if __has_include(<sanitizer/asan_interface.h>)
# include <sanitizer/asan_interface.h>
#endif
#include <Core/Defines.h>
#include <Common/memcpySmall.h>
#include <Common/ProfileEvents.h>

View File

@ -1,6 +1,9 @@
#pragma once
#include <sanitizer/asan_interface.h>
#if __has_include(<sanitizer/asan_interface.h>)
# include <sanitizer/asan_interface.h>
#endif
#include <Core/Defines.h>
#include <Common/Arena.h>
#include <Common/BitHelpers.h>

View File

@ -1,8 +1,6 @@
#include <Common/config.h>
#include "MiAllocator.h"
#if USE_MIMALLOC
#include "MiAllocator.h"
#include <mimalloc.h>
#include <Common/Exception.h>

View File

@ -2,10 +2,7 @@
#include <Common/config.h>
#if !USE_MIMALLOC
#error "do not include this file until USE_MIMALLOC is set to 1"
#endif
#if USE_MIMALLOC
#include <cstddef>
namespace DB
@ -26,3 +23,5 @@ struct MiAllocator
};
}
#endif

View File

@ -114,7 +114,7 @@ namespace
out.next();
}
const UInt32 TIMER_PRECISION = 1e9;
[[maybe_unused]] const UInt32 TIMER_PRECISION = 1e9;
}
namespace ErrorCodes
@ -158,7 +158,12 @@ QueryProfilerBase<ProfilerImpl>::QueryProfilerBase(const Int32 thread_id, const
struct sigevent sev;
sev.sigev_notify = SIGEV_THREAD_ID;
sev.sigev_signo = pause_signal;
#if defined(__FreeBSD__)
sev._sigev_un._threadid = thread_id;
#else
sev._sigev_un._tid = thread_id;
#endif
if (timer_create(clock_type, &sev, &timer_id))
throwFromErrno("Failed to create thread timer", ErrorCodes::CANNOT_CREATE_TIMER);

View File

@ -1,11 +1,10 @@
#include <common/SimpleCache.h>
#include <common/demangle.h>
#include <Common/config.h>
#include <Common/StackTrace.h>
#include <Common/SymbolIndex.h>
#include <Common/Dwarf.h>
#include <Common/Elf.h>
#include <sstream>
#include <filesystem>
#include <unordered_map>
@ -31,6 +30,8 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext
error << " Access: write.";
else
error << " Access: read.";
#else
UNUSED(context);
#endif
switch (info.si_code)

View File

@ -46,6 +46,7 @@ TraceCollector::TraceCollector(std::shared_ptr<TraceLog> & trace_log)
if (-1 == fcntl(trace_pipe.fds_rw[1], F_SETFL, flags | O_NONBLOCK))
throwFromErrno("Cannot set non-blocking mode of pipe", ErrorCodes::CANNOT_FCNTL);
#if !defined(__FreeBSD__)
/** Increase pipe size to avoid slowdown during fine-grained trace collection.
*/
constexpr int max_pipe_capacity_to_set = 1048576;
@ -57,6 +58,7 @@ TraceCollector::TraceCollector(std::shared_ptr<TraceLog> & trace_log)
throwFromErrno("Cannot increase pipe capacity to " + toString(pipe_size * 2), ErrorCodes::CANNOT_FCNTL);
LOG_TRACE(log, "Pipe capacity is " << formatReadableSizeWithBinarySuffix(std::min(pipe_size, max_pipe_capacity_to_set)));
#endif
thread = ThreadFromGlobalPool(&TraceCollector::run, this);
}

View File

@ -9,5 +9,5 @@
#cmakedefine01 USE_CPUINFO
#cmakedefine01 USE_BROTLI
#cmakedefine01 USE_MIMALLOC
#cmakedefine01 USE_UNWIND
#cmakedefine01 CLICKHOUSE_SPLIT_BINARY

View File

@ -1,11 +1,8 @@
#include "CompressedReadBufferBase.h"
#include <vector>
#include <string.h>
#include <city.h>
#include <zstd.h>
#include <Common/PODArray.h>
#include <Common/ProfileEvents.h>
#include <Common/Exception.h>

View File

@ -1,8 +1,5 @@
#include <memory>
#include <city.h>
#include <lz4.h>
#include <lz4hc.h>
#include <zstd.h>
#include <string.h>
#include <common/unaligned.h>

View File

@ -7,7 +7,6 @@
#include <IO/ReadBufferFromFileBase.h>
#include <Common/typeid_cast.h>
#include <Compression/CompressionFactory.h>
#include <zstd.h>
namespace ProfileEvents
{

View File

@ -140,6 +140,11 @@
/// It could be any magic number.
#define DBMS_DISTRIBUTED_SENDS_MAGIC_NUMBER 0xCAFECABE
#if !__has_include(<sanitizer/asan_interface.h>)
# define ASAN_UNPOISON_MEMORY_REGION(a, b)
# define ASAN_POISON_MEMORY_REGION(a, b)
#endif
/// A macro for suppressing warnings about unused variables or function results.
/// Useful for structured bindings which have no standard way to declare this.
#define UNUSED(...) (void)(__VA_ARGS__)

View File

@ -1,8 +1,6 @@
#include "config_formats.h"
#if USE_CAPNP
#include "CapnProtoRowInputStream.h"
#if USE_CAPNP
#include <IO/ReadBuffer.h>
#include <Interpreters/Context.h>
#include <Formats/FormatFactory.h>

View File

@ -1,7 +1,6 @@
#include "config_formats.h"
#if USE_PARQUET
# include "ParquetBlockInputStream.h"
#include "ParquetBlockInputStream.h"
#if USE_PARQUET
# include <algorithm>
# include <iterator>
# include <vector>

View File

@ -1,7 +1,6 @@
#include "config_formats.h"
#if USE_PARQUET
# include "ParquetBlockOutputStream.h"
#include "ParquetBlockOutputStream.h"
#if USE_PARQUET
// TODO: clean includes
# include <Columns/ColumnDecimal.h>
# include <Columns/ColumnFixedString.h>

View File

@ -1,8 +1,5 @@
#include "config_formats.h"
#if USE_PROTOBUF
#include "ProtobufColumnMatcher.h"
#if USE_PROTOBUF
#include <Common/Exception.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/descriptor.pb.h>

View File

@ -9,7 +9,7 @@
#if USE_PROTOBUF
#include <boost/noncopyable.hpp>
#include <Formats/ProtobufColumnMatcher.h>
#include "ProtobufColumnMatcher.h"
#include <IO/ReadBuffer.h>
#include <memory>

View File

@ -1,8 +1,6 @@
#include "config_formats.h"
#if USE_PROTOBUF
#include "ProtobufRowInputStream.h"
#if USE_PROTOBUF
#include <Core/Block.h>
#include <Formats/BlockInputStreamFromRowInputStream.h>
#include <Formats/FormatFactory.h>

View File

@ -7,7 +7,7 @@
#include "config_formats.h"
#if USE_PROTOBUF
#include <Formats/ProtobufColumnMatcher.h>
#include "ProtobufColumnMatcher.h"
#include <IO/WriteBufferFromString.h>
#include <boost/noncopyable.hpp>
#include <Common/PODArray.h>

View File

@ -22,6 +22,7 @@ target_link_libraries(clickhouse_functions
${BASE64_LIBRARY}
PRIVATE
${ZLIB_LIBRARIES}
${Boost_FILESYSTEM_LIBRARY}
)

View File

@ -254,7 +254,7 @@ void ReadBufferAIO::prepare()
/// Region of the disk from which we want to read data.
const off_t region_begin = first_unread_pos_in_file;
if ((requested_byte_count > std::numeric_limits<off_t>::max()) ||
if ((requested_byte_count > static_cast<size_t>(std::numeric_limits<off_t>::max())) ||
(first_unread_pos_in_file > (std::numeric_limits<off_t>::max() - static_cast<off_t>(requested_byte_count))))
throw Exception("An overflow occurred during file operation", ErrorCodes::LOGICAL_ERROR);

View File

@ -1,8 +1,6 @@
#include <Common/config.h>
#include "ReadBufferFromHDFS.h"
#if USE_HDFS
#include <IO/ReadBufferFromHDFS.h>
#include <IO/HDFSCommon.h>
#include <Poco/URI.h>
#include <hdfs/hdfs.h>

View File

@ -1,5 +1,7 @@
#pragma once
#include <Common/config.h>
#if USE_HDFS
#include <IO/ReadBuffer.h>
#include <IO/BufferWithOwnMemory.h>

View File

@ -274,7 +274,7 @@ void WriteBufferAIO::prepare()
/// Region of the disk in which we want to write data.
const off_t region_begin = pos_in_file;
if ((flush_buffer.offset() > std::numeric_limits<off_t>::max()) ||
if ((flush_buffer.offset() > static_cast<size_t>(std::numeric_limits<off_t>::max())) ||
(pos_in_file > (std::numeric_limits<off_t>::max() - static_cast<off_t>(flush_buffer.offset()))))
throw Exception("An overflow occurred during file operation", ErrorCodes::LOGICAL_ERROR);