thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-11 17:02:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add cross-compile build for FreeBSD (#9643)

Browse Source 
* Add toolchain to Docker image
This commit is contained in:
Ivan 2020-04-07 11:33:49 +03:00 committed by GitHub
parent d300a7da3c
commit f6b31f344d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 181 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CMakeLists.txt
View File

@ -228,7 +228,7 @@ else ()
set(NOT_UNBUNDLED 1)
endif ()
if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN) OR ARCH_32)
if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN))
# Using system libs can cause a lot of warnings in includes (on macro expansion).
option (WERROR "Enable -Werror compiler option" OFF)
else ()
@ -251,6 +251,8 @@ if (OS_LINUX)
include(cmake/linux/default_libs.cmake)
elseif (OS_DARWIN)
include(cmake/darwin/default_libs.cmake)
elseif (OS_FREEBSD)
include(cmake/freebsd/default_libs.cmake)
endif ()
######################################
@ -316,7 +318,6 @@ include (cmake/find/poco.cmake)
include (cmake/find/lz4.cmake)
include (cmake/find/xxhash.cmake)
include (cmake/find/sparsehash.cmake)
include (cmake/find/execinfo.cmake)
include (cmake/find/re2.cmake)
include (cmake/find/libgsasl.cmake)
include (cmake/find/rdkafka.cmake)

2
base/common/time.h
View File

@ -4,4 +4,6 @@
#if defined (OS_DARWIN)
# define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC
#elif defined (OS_FREEBSD)
# define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC_FAST
#endif

1
cmake/arch.cmake
View File

@ -11,7 +11,6 @@ if (CMAKE_LIBRARY_ARCHITECTURE MATCHES "i386")
set (ARCH_I386 1)
endif ()
if ((ARCH_ARM AND NOT ARCH_AARCH64) OR ARCH_I386)
set (ARCH_32 1)
message (FATAL_ERROR "32bit platforms are not supported")
endif ()

8
cmake/find/execinfo.cmake
View File

@ -1,8 +0,0 @@
if (OS_FREEBSD)
find_library (EXECINFO_LIBRARY execinfo)
find_library (ELF_LIBRARY elf)
set (EXECINFO_LIBRARIES ${EXECINFO_LIBRARY} ${ELF_LIBRARY})
message (STATUS "Using execinfo: ${EXECINFO_LIBRARIES}")
else ()
set (EXECINFO_LIBRARIES "")
endif ()

4
cmake/find/libgsasl.cmake
View File

@ -1,6 +1,4 @@
if (NOT ARCH_32)
option (USE_INTERNAL_LIBGSASL_LIBRARY "Set to FALSE to use system libgsasl library instead of bundled" ${NOT_UNBUNDLED})
endif ()
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src/gsasl.h")
if (USE_INTERNAL_LIBGSASL_LIBRARY)
@ -16,7 +14,7 @@ if (NOT USE_INTERNAL_LIBGSASL_LIBRARY)
endif ()
if (LIBGSASL_LIBRARY AND LIBGSASL_INCLUDE_DIR)
elseif (NOT MISSING_INTERNAL_LIBGSASL_LIBRARY AND NOT ARCH_32)
elseif (NOT MISSING_INTERNAL_LIBGSASL_LIBRARY)
set (LIBGSASL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src ${ClickHouse_SOURCE_DIR}/contrib/libgsasl/linux_x86_64/include)
set (USE_INTERNAL_LIBGSASL_LIBRARY 1)
set (LIBGSASL_LIBRARY libgsasl)

2
cmake/find/rdkafka.cmake
View File

@ -1,5 +1,5 @@
# Freebsd: contrib/cppkafka/include/cppkafka/detail/endianness.h:53:23: error: 'betoh16' was not declared in this scope
if (NOT ARCH_ARM AND NOT ARCH_32 AND NOT OS_FREEBSD AND OPENSSL_FOUND)
if (NOT ARCH_ARM AND NOT OS_FREEBSD AND OPENSSL_FOUND)
option (ENABLE_RDKAFKA "Enable kafka" ${ENABLE_LIBRARIES})
endif ()

2
cmake/find/ssl.cmake
View File

@ -2,9 +2,7 @@ option(ENABLE_SSL "Enable ssl" ${ENABLE_LIBRARIES})
if(ENABLE_SSL)
if(NOT ARCH_32)
option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ${NOT_UNBUNDLED})
endif()
if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/openssl/README")
if(USE_INTERNAL_SSL_LIBRARY)

9
cmake/find/unwind.cmake
View File

@ -1,14 +1,5 @@
option (USE_UNWIND "Enable libunwind (better stacktraces)" ${ENABLE_LIBRARIES})
if (NOT CMAKE_SYSTEM MATCHES "Linux" OR ARCH_ARM OR ARCH_32)
set (USE_UNWIND OFF)
endif ()
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libunwind/CMakeLists.txt")
message(WARNING "submodule contrib/libunwind is missing. to fix try run: \n git submodule update --init --recursive")
set (USE_UNWIND OFF)
endif ()
if (USE_UNWIND)
add_subdirectory(contrib/libunwind-cmake)
set (UNWIND_LIBRARIES unwind)

2
cmake/find/zlib.cmake
View File

@ -1,6 +1,4 @@
if (NOT OS_FREEBSD AND NOT ARCH_32)
option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED})
endif ()
if (NOT MSVC)
set (INTERNAL_ZLIB_NAME "zlib-ng" CACHE INTERNAL "")

40
cmake/freebsd/default_libs.cmake Normal file
View File

@ -0,0 +1,40 @@
set (DEFAULT_LIBS "-nodefaultlibs")
if (NOT COMPILER_CLANG)
message (FATAL_ERROR "FreeBSD build is supported only for Clang")
endif ()
execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread")
message(STATUS "Default libraries: ${DEFAULT_LIBS}")
set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
# Global libraries
add_library(global-libs INTERFACE)
# Unfortunately '-pthread' doesn't work with '-nodefaultlibs'.
# Just make sure we have pthreads at all.
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
include (cmake/find/unwind.cmake)
include (cmake/find/cxx.cmake)
add_library(global-group INTERFACE)
target_link_libraries(global-group INTERFACE
$<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>
)
link_libraries(global-group)
# FIXME: remove when all contribs will get custom cmake lists
install(
TARGETS global-group global-libs
EXPORT global
)

19
cmake/freebsd/toolchain-x86_64.cmake Normal file
View File

@ -0,0 +1,19 @@
set (CMAKE_SYSTEM_NAME "FreeBSD")
set (CMAKE_SYSTEM_PROCESSOR "x86_64")
set (CMAKE_C_COMPILER_TARGET "x86_64-pc-freebsd12.1")
set (CMAKE_CXX_COMPILER_TARGET "x86_64-pc-freebsd12.1")
set (CMAKE_ASM_COMPILER_TARGET "x86_64-pc-freebsd12.1")
set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/freebsd-x86_64")
set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake
set (LINKER_NAME "lld" CACHE STRING "" FORCE)
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

3
cmake/target.cmake
View File

@ -24,6 +24,9 @@ if (CMAKE_CROSSCOMPILING)
set (ENABLE_PARQUET OFF CACHE INTERNAL "")
set (ENABLE_MYSQL OFF CACHE INTERNAL "")
endif ()
elseif (OS_FREEBSD)
# FIXME: broken dependencies
set (ENABLE_PROTOBUF OFF CACHE INTERNAL "")
else ()
message (FATAL_ERROR "Trying to cross-compile to unsupported system: ${CMAKE_SYSTEM_NAME}!")
endif ()

3
docker/packager/binary/Dockerfile
View File

@ -65,5 +65,8 @@ RUN wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/M
# It contains all required headers and libraries. Note that it's named as "gcc" but actually we are using clang for cross compiling.
RUN wget "https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en" -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz
# Download toolchain for FreeBSD 12.1
RUN wget https://clickhouse-datasets.s3.yandex.net/toolchains/toolchains/freebsd-12.1-toolchain.tar.xz
COPY build.sh /
CMD ["/bin/bash", "/build.sh"]

3
docker/packager/binary/build.sh
View File

@ -8,6 +8,9 @@ tar xJf MacOSX10.14.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-co
mkdir -p build/cmake/toolchain/linux-aarch64
tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build/cmake/toolchain/linux-aarch64 --strip-components=1
mkdir -p build/cmake/toolchain/freebsd-x86_64
tar xJf freebsd-12.1-toolchain.tar.xz -C build/cmake/toolchain/freebsd-x86_64 --strip-components=1
mkdir -p build/build_docker
cd build/build_docker
ccache --show-stats ||:

7
docker/packager/packager
View File

@ -107,6 +107,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
CLANG_PREFIX = "clang"
DARWIN_SUFFIX = "-darwin"
ARM_SUFFIX = "-aarch64"
FREEBSD_SUFFIX = "-freebsd"
result = []
cmake_flags = ['$CMAKE_FLAGS', '-DADD_GDB_INDEX_FOR_GOLD=1']
@ -114,7 +115,8 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
is_clang = compiler.startswith(CLANG_PREFIX)
is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
is_cross_arm = compiler.endswith(ARM_SUFFIX)
is_cross_compile = is_cross_darwin or is_cross_arm
is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
is_cross_compile = is_cross_darwin or is_cross_arm or is_cross_freebsd
# Explicitly use LLD with Clang by default.
# Don't force linker for cross-compilation.
@ -131,6 +133,9 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
elif is_cross_arm:
cc = compiler[:-len(ARM_SUFFIX)]
cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake")
elif is_cross_freebsd:
cc = compiler[:-len(FREEBSD_SUFFIX)]
cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake")
else:
cc = compiler

4
src/CMakeLists.txt
View File

@ -274,10 +274,6 @@ set_source_files_properties(Common/malloc.cpp PROPERTIES COMPILE_FLAGS "-fno-bui
add_library (clickhouse_new_delete STATIC Common/new_delete.cpp)
target_link_libraries (clickhouse_new_delete PRIVATE clickhouse_common_io jemalloc)
if (OS_FREEBSD)
target_compile_definitions (clickhouse_common_io PUBLIC CLOCK_MONOTONIC_COARSE=CLOCK_MONOTONIC_FAST)
endif ()
add_subdirectory(Common/ZooKeeper)
add_subdirectory(Common/Config)

5
src/Common/QueryProfiler.cpp
View File

@ -23,7 +23,9 @@ namespace DB
namespace
{
#if defined(OS_LINUX)
thread_local size_t write_trace_iteration = 0;
#endif
void writeTraceInfo(TraceType trace_type, int /* sig */, siginfo_t * info, void * context)
{
@ -53,7 +55,6 @@ namespace
}
#else
UNUSED(info);
UNUSED(write_trace_iteration);
#endif
const auto signal_context = *reinterpret_cast<ucontext_t *>(context);
@ -110,7 +111,7 @@ QueryProfilerBase<ProfilerImpl>::QueryProfilerBase(const UInt64 thread_id, const
sev.sigev_notify = SIGEV_THREAD_ID;
sev.sigev_signo = pause_signal;
# if defined(__FreeBSD__)
# if defined(OS_FREEBSD)
sev._sigev_un._threadid = thread_id;
# else
sev._sigev_un._tid = thread_id;

31
src/Common/ThreadFuzzer.cpp
View File

@ -76,7 +76,7 @@ static std::atomic<int> num_cpus = 0;
static std::atomic<double> NAME##_after_yield_probability = 0; \
static std::atomic<double> NAME##_after_migrate_probability = 0; \
static std::atomic<double> NAME##_after_sleep_probability = 0; \
static std::atomic<double> NAME ## _after_sleep_time_us = 0; \
static std::atomic<double> NAME##_after_sleep_time_us = 0;
FOR_EACH_WRAPPED_FUNCTION(DEFINE_WRAPPER_PARAMS)
@ -107,7 +107,7 @@ void ThreadFuzzer::initConfiguration()
initFromEnv(NAME##_after_yield_probability, "THREAD_FUZZER_" #NAME "_AFTER_YIELD_PROBABILITY"); \
initFromEnv(NAME##_after_migrate_probability, "THREAD_FUZZER_" #NAME "_AFTER_MIGRATE_PROBABILITY"); \
initFromEnv(NAME##_after_sleep_probability, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_PROBABILITY"); \
initFromEnv(NAME ## _after_sleep_time_us, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US"); \
initFromEnv(NAME##_after_sleep_time_us, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US");
FOR_EACH_WRAPPED_FUNCTION(INIT_WRAPPER_PARAMS)
@ -120,15 +120,23 @@ bool ThreadFuzzer::isEffective() const
{
#if defined(OS_LINUX)
# define CHECK_WRAPPER_PARAMS(RET, NAME, ...) \
if (NAME ## _before_yield_probability.load(std::memory_order_relaxed)) return true; \
if (NAME ## _before_migrate_probability.load(std::memory_order_relaxed)) return true; \
if (NAME ## _before_sleep_probability.load(std::memory_order_relaxed)) return true; \
if (NAME ## _before_sleep_time_us.load(std::memory_order_relaxed)) return true; \
if (NAME##_before_yield_probability.load(std::memory_order_relaxed)) \
return true; \
if (NAME##_before_migrate_probability.load(std::memory_order_relaxed)) \
return true; \
if (NAME##_before_sleep_probability.load(std::memory_order_relaxed)) \
return true; \
if (NAME##_before_sleep_time_us.load(std::memory_order_relaxed)) \
return true; \
\
if (NAME ## _after_yield_probability.load(std::memory_order_relaxed)) return true; \
if (NAME ## _after_migrate_probability.load(std::memory_order_relaxed)) return true; \
if (NAME ## _after_sleep_probability.load(std::memory_order_relaxed)) return true; \
if (NAME ## _after_sleep_time_us.load(std::memory_order_relaxed)) return true; \
if (NAME##_after_yield_probability.load(std::memory_order_relaxed)) \
return true; \
if (NAME##_after_migrate_probability.load(std::memory_order_relaxed)) \
return true; \
if (NAME##_after_sleep_probability.load(std::memory_order_relaxed)) \
return true; \
if (NAME##_after_sleep_time_us.load(std::memory_order_relaxed)) \
return true;
FOR_EACH_WRAPPED_FUNCTION(CHECK_WRAPPER_PARAMS)
@ -248,11 +256,10 @@ void ThreadFuzzer::setup()
NAME##_after_sleep_time_us.load(std::memory_order_relaxed)); \
\
return ret; \
} \
}
FOR_EACH_WRAPPED_FUNCTION(MAKE_WRAPPER)
# undef MAKE_WRAPPER
#endif
}

7
src/Common/setThreadName.cpp
View File

@ -29,11 +29,10 @@ void setThreadName(const char * name)
throw DB::Exception("Thread name cannot be longer than 15 bytes", DB::ErrorCodes::PTHREAD_ERROR);
#endif
#if defined(__FreeBSD__)
#if defined(OS_FREEBSD)
pthread_set_name_np(pthread_self(), name);
return;
#elif defined(__APPLE__)
if ((false))
#elif defined(OS_DARWIN)
if (0 != pthread_setname_np(name))
#else
if (0 != prctl(PR_SET_NAME, name, 0, 0, 0))

19
src/IO/AIO.cpp
View File

@ -1,12 +1,12 @@
#if defined(__linux__)
#include <IO/AIO.h>
#if defined(OS_LINUX)
#include <boost/noncopyable.hpp>
# include <Common/Exception.h>
# include <sys/syscall.h>
# include <unistd.h>
#include <IO/AIO.h>
/** Small wrappers for asynchronous I/O.
*/
@ -53,17 +53,10 @@ AIOContext::~AIOContext()
io_destroy(ctx);
}
#elif defined(__FreeBSD__)
#elif defined(OS_FREEBSD)
# include <aio.h>
# include <boost/noncopyable.hpp>
# include <sys/event.h>
# include <sys/time.h>
# include <sys/types.h>
# include <Common/Exception.h>
# include <IO/AIO.h>
/** Small wrappers for asynchronous I/O.
*/
@ -123,7 +116,7 @@ int io_submit(int ctx, long nr, struct iocb * iocbpp[])
int io_getevents(int ctx, long, long max_nr, struct kevent * events, struct timespec * timeout)
{
return kevent(ctx, NULL, 0, events, max_nr, timeout);
return kevent(ctx, nullptr, 0, events, max_nr, timeout);
}

6
src/IO/AIO.h
View File

@ -2,7 +2,7 @@
#include <boost/noncopyable.hpp>
#if defined(__linux__)
#if defined(OS_LINUX)
/// https://stackoverflow.com/questions/20759750/resolving-redefinition-of-timespec-in-time-h
# define timespec linux_timespec
@ -39,12 +39,12 @@ struct AIOContext : private boost::noncopyable
~AIOContext();
};
#elif defined(__FreeBSD__)
#elif defined(OS_FREEBSD)
# include <aio.h>
#include <sys/types.h>
# include <sys/event.h>
# include <sys/time.h>
# include <sys/types.h>
typedef struct kevent io_event;
typedef int aio_context_t;