Add cross-compile build for FreeBSD (#9643)

* Add toolchain to Docker image

CMakeLists.txt

@@ -228,7 +228,7 @@ else ()
     set(NOT_UNBUNDLED 1)
 endif ()
 
-if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN) OR ARCH_32)
+if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN))
     # Using system libs can cause a lot of warnings in includes (on macro expansion).
     option (WERROR "Enable -Werror compiler option" OFF)
 else ()
@@ -251,6 +251,8 @@ if (OS_LINUX)
     include(cmake/linux/default_libs.cmake)
 elseif (OS_DARWIN)
     include(cmake/darwin/default_libs.cmake)
+elseif (OS_FREEBSD)
+    include(cmake/freebsd/default_libs.cmake)
 endif ()
 
 ######################################
@@ -316,7 +318,6 @@ include (cmake/find/poco.cmake)
 include (cmake/find/lz4.cmake)
 include (cmake/find/xxhash.cmake)
 include (cmake/find/sparsehash.cmake)
-include (cmake/find/execinfo.cmake)
 include (cmake/find/re2.cmake)
 include (cmake/find/libgsasl.cmake)
 include (cmake/find/rdkafka.cmake)

base/common/time.h

@@ -4,4 +4,6 @@
 
 #if defined (OS_DARWIN)
 #    define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC
+#elif defined (OS_FREEBSD)
+#    define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC_FAST
 #endif

cmake/arch.cmake

@@ -11,7 +11,6 @@ if (CMAKE_LIBRARY_ARCHITECTURE MATCHES "i386")
     set (ARCH_I386 1)
 endif ()
 if ((ARCH_ARM AND NOT ARCH_AARCH64) OR ARCH_I386)
-    set (ARCH_32 1)
     message (FATAL_ERROR "32bit platforms are not supported")
 endif ()
 

cmake/find/execinfo.cmake

@@ -1,8 +0,0 @@
-if (OS_FREEBSD)
-    find_library (EXECINFO_LIBRARY execinfo)
-    find_library (ELF_LIBRARY elf)
-    set (EXECINFO_LIBRARIES ${EXECINFO_LIBRARY} ${ELF_LIBRARY})
-    message (STATUS "Using execinfo: ${EXECINFO_LIBRARIES}")
-else ()
-    set (EXECINFO_LIBRARIES "")
-endif ()

cmake/find/libgsasl.cmake

@@ -1,6 +1,4 @@
-if (NOT ARCH_32)
 option (USE_INTERNAL_LIBGSASL_LIBRARY "Set to FALSE to use system libgsasl library instead of bundled" ${NOT_UNBUNDLED})
-endif ()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src/gsasl.h")
     if (USE_INTERNAL_LIBGSASL_LIBRARY)
@@ -16,7 +14,7 @@ if (NOT USE_INTERNAL_LIBGSASL_LIBRARY)
 endif ()
 
 if (LIBGSASL_LIBRARY AND LIBGSASL_INCLUDE_DIR)
-elseif (NOT MISSING_INTERNAL_LIBGSASL_LIBRARY AND NOT ARCH_32)
+elseif (NOT MISSING_INTERNAL_LIBGSASL_LIBRARY)
     set (LIBGSASL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src ${ClickHouse_SOURCE_DIR}/contrib/libgsasl/linux_x86_64/include)
     set (USE_INTERNAL_LIBGSASL_LIBRARY 1)
     set (LIBGSASL_LIBRARY libgsasl)

cmake/find/rdkafka.cmake

@@ -1,5 +1,5 @@
 # Freebsd: contrib/cppkafka/include/cppkafka/detail/endianness.h:53:23: error: 'betoh16' was not declared in this scope
-if (NOT ARCH_ARM AND NOT ARCH_32 AND NOT OS_FREEBSD AND OPENSSL_FOUND)
+if (NOT ARCH_ARM AND NOT OS_FREEBSD AND OPENSSL_FOUND)
     option (ENABLE_RDKAFKA "Enable kafka" ${ENABLE_LIBRARIES})
 endif ()
 

cmake/find/ssl.cmake

@@ -2,9 +2,7 @@ option(ENABLE_SSL "Enable ssl" ${ENABLE_LIBRARIES})
 
 if(ENABLE_SSL)
 
-if(NOT ARCH_32)
 option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ${NOT_UNBUNDLED})
-endif()
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/openssl/README")
     if(USE_INTERNAL_SSL_LIBRARY)

cmake/find/unwind.cmake

@@ -1,14 +1,5 @@
 option (USE_UNWIND "Enable libunwind (better stacktraces)" ${ENABLE_LIBRARIES})
 
-if (NOT CMAKE_SYSTEM MATCHES "Linux" OR ARCH_ARM OR ARCH_32)
-    set (USE_UNWIND OFF)
-endif ()
-
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libunwind/CMakeLists.txt")
-    message(WARNING "submodule contrib/libunwind is missing. to fix try run: \n git submodule update --init --recursive")
-    set (USE_UNWIND OFF)
-endif ()
-
 if (USE_UNWIND)
     add_subdirectory(contrib/libunwind-cmake)
     set (UNWIND_LIBRARIES unwind)

cmake/find/zlib.cmake

@@ -1,6 +1,4 @@
-if (NOT OS_FREEBSD AND NOT ARCH_32)
 option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED})
-endif ()
 
 if (NOT MSVC)
     set (INTERNAL_ZLIB_NAME "zlib-ng" CACHE INTERNAL "")

cmake/freebsd/default_libs.cmake

@@ -0,0 +1,40 @@
+set (DEFAULT_LIBS "-nodefaultlibs")
+
+if (NOT COMPILER_CLANG)
+    message (FATAL_ERROR "FreeBSD build is supported only for Clang")
+endif ()
+
+execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread")
+
+message(STATUS "Default libraries: ${DEFAULT_LIBS}")
+
+set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
+set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
+
+# Global libraries
+
+add_library(global-libs INTERFACE)
+
+# Unfortunately '-pthread' doesn't work with '-nodefaultlibs'.
+# Just make sure we have pthreads at all.
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+
+include (cmake/find/unwind.cmake)
+include (cmake/find/cxx.cmake)
+
+add_library(global-group INTERFACE)
+
+target_link_libraries(global-group INTERFACE
+    $<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>
+)
+
+link_libraries(global-group)
+
+# FIXME: remove when all contribs will get custom cmake lists
+install(
+    TARGETS global-group global-libs
+    EXPORT global
+)

cmake/freebsd/toolchain-x86_64.cmake

@@ -0,0 +1,19 @@
+set (CMAKE_SYSTEM_NAME "FreeBSD")
+set (CMAKE_SYSTEM_PROCESSOR "x86_64")
+set (CMAKE_C_COMPILER_TARGET "x86_64-pc-freebsd12.1")
+set (CMAKE_CXX_COMPILER_TARGET "x86_64-pc-freebsd12.1")
+set (CMAKE_ASM_COMPILER_TARGET "x86_64-pc-freebsd12.1")
+set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/freebsd-x86_64")
+
+set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)  # disable linkage check - it doesn't work in CMake
+
+set (LINKER_NAME "lld" CACHE STRING "" FORCE)
+
+set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
+set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
+
+set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
+
+set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

cmake/target.cmake

@@ -24,6 +24,9 @@ if (CMAKE_CROSSCOMPILING)
             set (ENABLE_PARQUET OFF CACHE INTERNAL "")
             set (ENABLE_MYSQL OFF CACHE INTERNAL "")
         endif ()
+    elseif (OS_FREEBSD)
+        # FIXME: broken dependencies
+        set (ENABLE_PROTOBUF OFF CACHE INTERNAL "")
     else ()
         message (FATAL_ERROR "Trying to cross-compile to unsupported system: ${CMAKE_SYSTEM_NAME}!")
     endif ()

docker/packager/binary/Dockerfile

@@ -65,5 +65,8 @@ RUN wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/M
 # It contains all required headers and libraries. Note that it's named as "gcc" but actually we are using clang for cross compiling.
 RUN wget "https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en" -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz
 
+# Download toolchain for FreeBSD 12.1
+RUN wget https://clickhouse-datasets.s3.yandex.net/toolchains/toolchains/freebsd-12.1-toolchain.tar.xz
+
 COPY build.sh /
 CMD ["/bin/bash", "/build.sh"]

docker/packager/binary/build.sh

@@ -8,6 +8,9 @@ tar xJf MacOSX10.14.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-co
 mkdir -p build/cmake/toolchain/linux-aarch64
 tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build/cmake/toolchain/linux-aarch64 --strip-components=1
 
+mkdir -p build/cmake/toolchain/freebsd-x86_64
+tar xJf freebsd-12.1-toolchain.tar.xz -C build/cmake/toolchain/freebsd-x86_64 --strip-components=1
+
 mkdir -p build/build_docker
 cd build/build_docker
 ccache --show-stats ||:

docker/packager/packager

@@ -107,6 +107,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
     CLANG_PREFIX = "clang"
     DARWIN_SUFFIX = "-darwin"
     ARM_SUFFIX = "-aarch64"
+    FREEBSD_SUFFIX = "-freebsd"
 
     result = []
     cmake_flags = ['$CMAKE_FLAGS', '-DADD_GDB_INDEX_FOR_GOLD=1']
@@ -114,7 +115,8 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
     is_clang = compiler.startswith(CLANG_PREFIX)
     is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
     is_cross_arm = compiler.endswith(ARM_SUFFIX)
-    is_cross_compile = is_cross_darwin or is_cross_arm
+    is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
+    is_cross_compile = is_cross_darwin or is_cross_arm or is_cross_freebsd
 
     # Explicitly use LLD with Clang by default.
     # Don't force linker for cross-compilation.
@@ -131,6 +133,9 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
     elif is_cross_arm:
         cc = compiler[:-len(ARM_SUFFIX)]
         cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake")
+    elif is_cross_freebsd:
+        cc = compiler[:-len(FREEBSD_SUFFIX)]
+        cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake")
     else:
         cc = compiler
 

src/CMakeLists.txt

@@ -274,10 +274,6 @@ set_source_files_properties(Common/malloc.cpp PROPERTIES COMPILE_FLAGS "-fno-bui
 add_library (clickhouse_new_delete STATIC Common/new_delete.cpp)
 target_link_libraries (clickhouse_new_delete PRIVATE clickhouse_common_io jemalloc)
 
-if (OS_FREEBSD)
-    target_compile_definitions (clickhouse_common_io PUBLIC CLOCK_MONOTONIC_COARSE=CLOCK_MONOTONIC_FAST)
-endif ()
-
 add_subdirectory(Common/ZooKeeper)
 add_subdirectory(Common/Config)
 

src/Common/QueryProfiler.cpp

@@ -23,7 +23,9 @@ namespace DB
 
 namespace
 {
+#if defined(OS_LINUX)
     thread_local size_t write_trace_iteration = 0;
+#endif
 
     void writeTraceInfo(TraceType trace_type, int /* sig */, siginfo_t * info, void * context)
     {
@@ -53,7 +55,6 @@ namespace
         }
 #else
         UNUSED(info);
-        UNUSED(write_trace_iteration);
 #endif
 
         const auto signal_context = *reinterpret_cast<ucontext_t *>(context);
@@ -110,7 +111,7 @@ QueryProfilerBase<ProfilerImpl>::QueryProfilerBase(const UInt64 thread_id, const
         sev.sigev_notify = SIGEV_THREAD_ID;
         sev.sigev_signo = pause_signal;
 
-#   if defined(__FreeBSD__)
+#   if defined(OS_FREEBSD)
         sev._sigev_un._threadid = thread_id;
 #   else
         sev._sigev_un._tid = thread_id;

src/Common/ThreadFuzzer.cpp

@@ -76,7 +76,7 @@ static std::atomic<int> num_cpus = 0;
         static std::atomic<double> NAME##_after_yield_probability = 0; \
         static std::atomic<double> NAME##_after_migrate_probability = 0; \
         static std::atomic<double> NAME##_after_sleep_probability = 0; \
-    static std::atomic<double> NAME ## _after_sleep_time_us = 0; \
+        static std::atomic<double> NAME##_after_sleep_time_us = 0;
 
 FOR_EACH_WRAPPED_FUNCTION(DEFINE_WRAPPER_PARAMS)
 
@@ -107,7 +107,7 @@ void ThreadFuzzer::initConfiguration()
         initFromEnv(NAME##_after_yield_probability, "THREAD_FUZZER_" #NAME "_AFTER_YIELD_PROBABILITY"); \
         initFromEnv(NAME##_after_migrate_probability, "THREAD_FUZZER_" #NAME "_AFTER_MIGRATE_PROBABILITY"); \
         initFromEnv(NAME##_after_sleep_probability, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_PROBABILITY"); \
-    initFromEnv(NAME ## _after_sleep_time_us, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US"); \
+        initFromEnv(NAME##_after_sleep_time_us, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US");
 
     FOR_EACH_WRAPPED_FUNCTION(INIT_WRAPPER_PARAMS)
 
@@ -120,15 +120,23 @@ bool ThreadFuzzer::isEffective() const
 {
 #if defined(OS_LINUX)
 #    define CHECK_WRAPPER_PARAMS(RET, NAME, ...) \
-    if (NAME ## _before_yield_probability.load(std::memory_order_relaxed)) return true; \
+        if (NAME##_before_yield_probability.load(std::memory_order_relaxed)) \
-    if (NAME ## _before_migrate_probability.load(std::memory_order_relaxed)) return true; \
+            return true; \
-    if (NAME ## _before_sleep_probability.load(std::memory_order_relaxed)) return true; \
+        if (NAME##_before_migrate_probability.load(std::memory_order_relaxed)) \
-    if (NAME ## _before_sleep_time_us.load(std::memory_order_relaxed)) return true; \
+            return true; \
+        if (NAME##_before_sleep_probability.load(std::memory_order_relaxed)) \
+            return true; \
+        if (NAME##_before_sleep_time_us.load(std::memory_order_relaxed)) \
+            return true; \
 \
-    if (NAME ## _after_yield_probability.load(std::memory_order_relaxed)) return true; \
+        if (NAME##_after_yield_probability.load(std::memory_order_relaxed)) \
-    if (NAME ## _after_migrate_probability.load(std::memory_order_relaxed)) return true; \
+            return true; \
-    if (NAME ## _after_sleep_probability.load(std::memory_order_relaxed)) return true; \
+        if (NAME##_after_migrate_probability.load(std::memory_order_relaxed)) \
-    if (NAME ## _after_sleep_time_us.load(std::memory_order_relaxed)) return true; \
+            return true; \
+        if (NAME##_after_sleep_probability.load(std::memory_order_relaxed)) \
+            return true; \
+        if (NAME##_after_sleep_time_us.load(std::memory_order_relaxed)) \
+            return true;
 
     FOR_EACH_WRAPPED_FUNCTION(CHECK_WRAPPER_PARAMS)
 
@@ -248,11 +256,10 @@ void ThreadFuzzer::setup()
                 NAME##_after_sleep_time_us.load(std::memory_order_relaxed)); \
 \
             return ret; \
-    } \
+        }
 
 FOR_EACH_WRAPPED_FUNCTION(MAKE_WRAPPER)
 
 #    undef MAKE_WRAPPER
 #endif
-
 }

src/Common/setThreadName.cpp

@@ -29,11 +29,10 @@ void setThreadName(const char * name)
         throw DB::Exception("Thread name cannot be longer than 15 bytes", DB::ErrorCodes::PTHREAD_ERROR);
 #endif
 
-#if defined(__FreeBSD__)
+#if defined(OS_FREEBSD)
     pthread_set_name_np(pthread_self(), name);
-    return;
+    if ((false))
-
+#elif defined(OS_DARWIN)
-#elif defined(__APPLE__)
     if (0 != pthread_setname_np(name))
 #else
     if (0 != prctl(PR_SET_NAME, name, 0, 0, 0))

src/IO/AIO.cpp

@@ -1,12 +1,12 @@
-#if defined(__linux__)
+#include <IO/AIO.h>
+
+#if defined(OS_LINUX)
 
-#include <boost/noncopyable.hpp>
 #    include <Common/Exception.h>
+
 #    include <sys/syscall.h>
 #    include <unistd.h>
 
-#include <IO/AIO.h>
-
 
 /** Small wrappers for asynchronous I/O.
   */
@@ -53,17 +53,10 @@ AIOContext::~AIOContext()
     io_destroy(ctx);
 }
 
-#elif defined(__FreeBSD__)
+#elif defined(OS_FREEBSD)
 
-#    include <aio.h>
-#    include <boost/noncopyable.hpp>
-#    include <sys/event.h>
-#    include <sys/time.h>
-#    include <sys/types.h>
 #    include <Common/Exception.h>
 
-#    include <IO/AIO.h>
-
 
 /** Small wrappers for asynchronous I/O.
   */
@@ -123,7 +116,7 @@ int io_submit(int ctx, long nr, struct iocb * iocbpp[])
 
 int io_getevents(int ctx, long, long max_nr, struct kevent * events, struct timespec * timeout)
 {
-    return kevent(ctx, NULL, 0, events, max_nr, timeout);
+    return kevent(ctx, nullptr, 0, events, max_nr, timeout);
 }
 
 

src/IO/AIO.h

@@ -2,7 +2,7 @@
 
 #include <boost/noncopyable.hpp>
 
-#if defined(__linux__)
+#if defined(OS_LINUX)
 
 /// https://stackoverflow.com/questions/20759750/resolving-redefinition-of-timespec-in-time-h
 #    define timespec linux_timespec
@@ -39,12 +39,12 @@ struct AIOContext : private boost::noncopyable
     ~AIOContext();
 };
 
-#elif defined(__FreeBSD__)
+#elif defined(OS_FREEBSD)
 
 #    include <aio.h>
-#include <sys/types.h>
 #    include <sys/event.h>
 #    include <sys/time.h>
+#    include <sys/types.h>
 
 typedef struct kevent io_event;
 typedef int aio_context_t;