diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e82e18f823..35491fdcacd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,13 @@ endif () # Make sure the final executable has symbols exported set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic") +find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy") +if (OBJCOPY_PATH) + message(STATUS "Using objcopy: ${OBJCOPY_PATH}.") +else () + message(FATAL_ERROR "Cannot find objcopy.") +endif () + option (ADD_GDB_INDEX_FOR_GOLD "Set to add .gdb-index to resulting binaries for gold linker. NOOP if lld is used." 0) if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") if (LINKER_NAME STREQUAL "lld") diff --git a/base/common/CMakeLists.txt b/base/common/CMakeLists.txt index f09335f0ca0..903168a0dd4 100644 --- a/base/common/CMakeLists.txt +++ b/base/common/CMakeLists.txt @@ -17,6 +17,7 @@ set (SRCS sleep.cpp terminalColors.cpp errnoToString.cpp + getResource.cpp ) if (ENABLE_REPLXX) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 83a4d1e9149..50620e21b8f 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -3,11 +3,9 @@ #include #include #include -#include +#include #include -#include - #include #include #include @@ -213,19 +211,9 @@ namespace cctz_extension const std::string & name, const std::function(const std::string & name)> & fallback) { - std::string name_replaced = name; - std::replace(name_replaced.begin(), name_replaced.end(), '/', '_'); - std::replace(name_replaced.begin(), name_replaced.end(), '-', '_'); - - /// These are the names that are generated by "ld -r -b binary" - std::string symbol_name_data = "_binary_" + name_replaced + "_start"; - std::string symbol_name_size = "_binary_" + name_replaced + "_size"; - - const void * sym_data = dlsym(RTLD_DEFAULT, symbol_name_data.c_str()); - const void * sym_size = dlsym(RTLD_DEFAULT, symbol_name_size.c_str()); - - if (sym_data && sym_size) - return std::make_unique(static_cast(sym_data), unalignedLoad(&sym_size)); + std::string_view resource = getResource(name); + if (!resource.empty()) + return std::make_unique(resource.data(), resource.size()); return fallback(name); } diff --git a/base/common/getResource.cpp b/base/common/getResource.cpp new file mode 100644 index 00000000000..be7f205e7d1 --- /dev/null +++ b/base/common/getResource.cpp @@ -0,0 +1,24 @@ +#include "getResource.h" +#include "unaligned.h" +#include +#include + + +std::string_view getResource(std::string_view name) +{ + std::string name_replaced(name); + std::replace(name_replaced.begin(), name_replaced.end(), '/', '_'); + std::replace(name_replaced.begin(), name_replaced.end(), '-', '_'); + std::replace(name_replaced.begin(), name_replaced.end(), '.', '_'); + + /// These are the names that are generated by "ld -r -b binary" + std::string symbol_name_data = "_binary_" + name_replaced + "_start"; + std::string symbol_name_size = "_binary_" + name_replaced + "_size"; + + const void * sym_data = dlsym(RTLD_DEFAULT, symbol_name_data.c_str()); + const void * sym_size = dlsym(RTLD_DEFAULT, symbol_name_size.c_str()); + + if (sym_data && sym_size) + return { static_cast(sym_data), unalignedLoad(&sym_size) }; + return {}; +} diff --git a/base/common/getResource.h b/base/common/getResource.h new file mode 100644 index 00000000000..8975cc7841e --- /dev/null +++ b/base/common/getResource.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +/// Get resource from binary if exists. Otherwise return empty string view. +/// Resources are data that is embedded into executable at link time. +std::string_view getResource(std::string_view name); diff --git a/base/common/ya.make b/base/common/ya.make index b64ab93f2fc..caa0d067ef2 100644 --- a/base/common/ya.make +++ b/base/common/ya.make @@ -1,3 +1,4 @@ +# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it. LIBRARY() ADDINCL( @@ -35,8 +36,10 @@ SRCS( DateLUT.cpp DateLUTImpl.cpp demangle.cpp + errnoToString.cpp getFQDNOrHostName.cpp getMemoryAmount.cpp + getResource.cpp getThreadId.cpp JSON.cpp LineReader.cpp @@ -47,7 +50,7 @@ SRCS( shift10.cpp sleep.cpp terminalColors.cpp - errnoToString.cpp + ) END() diff --git a/base/common/ya.make.in b/base/common/ya.make.in new file mode 100644 index 00000000000..957ae4d68b7 --- /dev/null +++ b/base/common/ya.make.in @@ -0,0 +1,36 @@ +LIBRARY() + +ADDINCL( + GLOBAL clickhouse/base + GLOBAL contrib/libs/cctz/include +) + +CFLAGS (GLOBAL -DARCADIA_BUILD) + +CFLAGS (GLOBAL -DUSE_CPUID=1) +CFLAGS (GLOBAL -DUSE_JEMALLOC=0) +CFLAGS (GLOBAL -DUSE_RAPIDJSON=1) + +IF (OS_DARWIN) + CFLAGS (GLOBAL -DOS_DARWIN) +ELSEIF (OS_FREEBSD) + CFLAGS (GLOBAL -DOS_FREEBSD) +ELSEIF (OS_LINUX) + CFLAGS (GLOBAL -DOS_LINUX) +ENDIF () + +PEERDIR( + contrib/libs/cctz/src + contrib/libs/cxxsupp/libcxx-filesystem + contrib/libs/poco/Net + contrib/libs/poco/Util + contrib/libs/fmt + contrib/restricted/boost + contrib/restricted/cityhash-1.0.2 +) + +SRCS( + +) + +END() diff --git a/contrib/cctz-cmake/CMakeLists.txt b/contrib/cctz-cmake/CMakeLists.txt index 0837a366f20..7eb768b8de9 100644 --- a/contrib/cctz-cmake/CMakeLists.txt +++ b/contrib/cctz-cmake/CMakeLists.txt @@ -28,13 +28,6 @@ if (USE_INTERNAL_CCTZ) if (OS_LINUX AND ARCH_AMD64) - find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy") - if (OBJCOPY_PATH) - message(STATUS "Using objcopy: ${OBJCOPY_PATH}.") - else () - message(FATAL_ERROR "Cannot find objcopy.") - endif () - set (TIMEZONES Africa/Abidjan Africa/Accra diff --git a/programs/server/CMakeLists.txt b/programs/server/CMakeLists.txt index 1563f5ac51e..fbb5eef446d 100644 --- a/programs/server/CMakeLists.txt +++ b/programs/server/CMakeLists.txt @@ -3,6 +3,10 @@ set(CLICKHOUSE_SERVER_SOURCES Server.cpp ) +if (OS_LINUX AND ARCH_AMD64) + set (LINK_CONFIG_LIB INTERFACE "-Wl,--whole-archive $ -Wl,--no-whole-archive") +endif () + set (CLICKHOUSE_SERVER_LINK PRIVATE clickhouse_aggregate_functions @@ -16,18 +20,42 @@ set (CLICKHOUSE_SERVER_LINK clickhouse_table_functions string_utils + ${LINK_CONFIG_LIB} + PUBLIC daemon ) clickhouse_program_add(server) -if (GLIBC_COMPATIBILITY) - set (GLIBC_MAX_REQUIRED 2.4 CACHE INTERNAL "") - # temporary disabled. to enable - change 'exit 0' to 'exit $a' - add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-server | perl -nE 'END {exit 0 if $a} ++$a, print if /\\x40GLIBC_(\\S+)/ and pack(q{C*}, split /\\./, \$1) gt pack q{C*}, split /\\./, q{${GLIBC_MAX_REQUIRED}}'") - - #add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-server | grep '@GLIBC' | grep -oP 'GLIBC_[\\d\\.]+' | sort | uniq | sort --version-sort --reverse | perl -lnE 'warn($_), exit 1 if $_ gt q{GLIBC_${GLIBC_MAX_REQUIRED}}'") # old -endif () - install(FILES config.xml users.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server COMPONENT clickhouse) + +# TODO We actually need this on Mac, FreeBSD and AArch64. +if (OS_LINUX AND ARCH_AMD64) + # Embed default config files as a resource into the binary. + # This is needed for two purposes: + # 1. Allow to run the binary without download of any other files. + # 2. Allow to implement "sudo clickhouse install" tool. + + foreach(CONFIG_FILE config users embedded) + set(CONFIG_OBJ ${CONFIG_FILE}.o) + set(CONFIG_OBJS ${CONFIG_OBJS} ${CONFIG_OBJ}) + + # https://stackoverflow.com/questions/14776463/compile-and-add-an-object-file-from-a-binary-with-cmake + add_custom_command(OUTPUT ${CONFIG_OBJ} + COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${OBJCOPY_PATH} -I binary -O elf64-x86-64 -B i386 ${CONFIG_FILE}.xml ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_OBJ} + COMMAND ${OBJCOPY_PATH} --rename-section .data=.rodata,alloc,load,readonly,data,contents + ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_OBJ} ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_OBJ}) + + set_source_files_properties(${CONFIG_OBJ} PROPERTIES EXTERNAL_OBJECT true GENERATED true) + endforeach(CONFIG_FILE) + + add_library(clickhouse_server_configs STATIC ${CONFIG_OBJS}) + set_target_properties(clickhouse_server_configs PROPERTIES LINKER_LANGUAGE C) + + # whole-archive prevents symbols from being discarded for unknown reason + # CMake can shuffle each of target_link_libraries arguments with other + # libraries in linker command. To avoid this we hardcode whole-archive + # library into single string. + add_dependencies(clickhouse-server-lib clickhouse_server_configs) +endif () diff --git a/programs/server/config.xml b/programs/server/config.xml index 506e7d0b006..af01e880dc2 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -154,7 +154,7 @@ of the time, in which case a higher number of threads might be required. --> - 10000 + 10000 diff --git a/programs/server/embedded.xml b/programs/server/embedded.xml new file mode 100644 index 00000000000..1ac568682f9 --- /dev/null +++ b/programs/server/embedded.xml @@ -0,0 +1,40 @@ + + + + + trace + true + + + 8123 + 9000 + 9004 + + ./ + + 8589934592 + 5368709120 + true + + + + + + + ::/0 + + + default + default + 1 + + + + + + + + + + + diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 0e36c6ee660..ab25ec35672 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -14,6 +15,8 @@ #include #include #include +#include +#include #define PREPROCESSED_SUFFIX "-preprocessed" @@ -23,6 +26,11 @@ using namespace Poco::XML; namespace DB { +namespace ErrorCodes +{ + extern const int FILE_DOESNT_EXIST; +} + /// For cutting preprocessed path to this base static std::string main_config_path; @@ -440,9 +448,27 @@ XMLDocumentPtr ConfigProcessor::processConfig( zkutil::ZooKeeperNodeCache * zk_node_cache, const zkutil::EventPtr & zk_changed_event) { + XMLDocumentPtr config; LOG_DEBUG(log, "Processing configuration file '{}'.", path); - XMLDocumentPtr config = dom_parser.parse(path); + if (std::filesystem::exists(path)) + { + config = dom_parser.parse(path); + } + else + { + /// When we can use config embedded in binary. + if (path == "config.xml") + { + auto resource = getResource("embedded.xml"); + if (resource.empty()) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Configuration file {} doesn't exist and there is no embedded config", path); + LOG_DEBUG(log, "There is no file '{}', will use embedded config.", path); + config = dom_parser.parseMemory(resource.data(), resource.size()); + } + else + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Configuration file {} doesn't exist", path); + } std::vector contributing_files; contributing_files.push_back(path); diff --git a/src/Disks/ya.make b/src/Disks/ya.make index f82cc4e2868..a64e5508ca8 100644 --- a/src/Disks/ya.make +++ b/src/Disks/ya.make @@ -7,6 +7,8 @@ PEERDIR( SRCS( createVolume.cpp + DiskCacheWrapper.cpp + DiskDecorator.cpp DiskFactory.cpp DiskLocal.cpp DiskMemory.cpp