Merge pull request #13515 from ClickHouse/embed-configs

Embed configs into binary

CMakeLists.txt

@@ -142,6 +142,13 @@ endif ()
 # Make sure the final executable has symbols exported
 set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic")
 
+find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy")
+if (OBJCOPY_PATH)
+    message(STATUS "Using objcopy: ${OBJCOPY_PATH}.")
+else ()
+    message(FATAL_ERROR "Cannot find objcopy.")
+endif ()
+
 option (ADD_GDB_INDEX_FOR_GOLD "Set to add .gdb-index to resulting binaries for gold linker. NOOP if lld is used." 0)
 if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
     if (LINKER_NAME STREQUAL "lld")

base/common/CMakeLists.txt

@@ -17,6 +17,7 @@ set (SRCS
     sleep.cpp
     terminalColors.cpp
     errnoToString.cpp
+    getResource.cpp
 )
 
 if (ENABLE_REPLXX)

base/common/DateLUTImpl.cpp

@@ -3,11 +3,9 @@
 #include <cctz/civil_time.h>
 #include <cctz/time_zone.h>
 #include <cctz/zone_info_source.h>
-#include <common/unaligned.h>
+#include <common/getResource.h>
 #include <Poco/Exception.h>
 
-#include <dlfcn.h>
-
 #include <algorithm>
 #include <cassert>
 #include <chrono>
@@ -213,19 +211,9 @@ namespace cctz_extension
             const std::string & name,
             const std::function<std::unique_ptr<cctz::ZoneInfoSource>(const std::string & name)> & fallback)
         {
-            std::string name_replaced = name;
-            std::replace(name_replaced.begin(), name_replaced.end(), '/', '_');
-            std::replace(name_replaced.begin(), name_replaced.end(), '-', '_');
-
-            /// These are the names that are generated by "ld -r -b binary"
-            std::string symbol_name_data = "_binary_" + name_replaced + "_start";
-            std::string symbol_name_size = "_binary_" + name_replaced + "_size";
-
-            const void * sym_data = dlsym(RTLD_DEFAULT, symbol_name_data.c_str());
-            const void * sym_size = dlsym(RTLD_DEFAULT, symbol_name_size.c_str());
-
-            if (sym_data && sym_size)
-                return std::make_unique<Source>(static_cast<const char *>(sym_data), unalignedLoad<size_t>(&sym_size));
+            std::string_view resource = getResource(name);
+            if (!resource.empty())
+                return std::make_unique<Source>(resource.data(), resource.size());
 
             return fallback(name);
         }

base/common/getResource.cpp

@@ -0,0 +1,24 @@
+#include "getResource.h"
+#include "unaligned.h"
+#include <dlfcn.h>
+#include <string>
+
+
+std::string_view getResource(std::string_view name)
+{
+    std::string name_replaced(name);
+    std::replace(name_replaced.begin(), name_replaced.end(), '/', '_');
+    std::replace(name_replaced.begin(), name_replaced.end(), '-', '_');
+    std::replace(name_replaced.begin(), name_replaced.end(), '.', '_');
+
+    /// These are the names that are generated by "ld -r -b binary"
+    std::string symbol_name_data = "_binary_" + name_replaced + "_start";
+    std::string symbol_name_size = "_binary_" + name_replaced + "_size";
+
+    const void * sym_data = dlsym(RTLD_DEFAULT, symbol_name_data.c_str());
+    const void * sym_size = dlsym(RTLD_DEFAULT, symbol_name_size.c_str());
+
+    if (sym_data && sym_size)
+        return { static_cast<const char *>(sym_data), unalignedLoad<size_t>(&sym_size) };
+    return {};
+}

base/common/getResource.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <string_view>
+
+/// Get resource from binary if exists. Otherwise return empty string view.
+/// Resources are data that is embedded into executable at link time.
+std::string_view getResource(std::string_view name);

base/common/ya.make

@@ -1,3 +1,4 @@
+# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
 LIBRARY()
 
 ADDINCL(
@@ -35,8 +36,10 @@ SRCS(
     DateLUT.cpp
     DateLUTImpl.cpp
     demangle.cpp
+    errnoToString.cpp
     getFQDNOrHostName.cpp
     getMemoryAmount.cpp
+    getResource.cpp
     getThreadId.cpp
     JSON.cpp
     LineReader.cpp
@@ -47,7 +50,7 @@ SRCS(
     shift10.cpp
     sleep.cpp
     terminalColors.cpp
-    errnoToString.cpp
+
 )
 
 END()

base/common/ya.make.in

@@ -0,0 +1,36 @@
+LIBRARY()
+
+ADDINCL(
+    GLOBAL clickhouse/base
+    GLOBAL contrib/libs/cctz/include
+)
+
+CFLAGS (GLOBAL -DARCADIA_BUILD)
+
+CFLAGS (GLOBAL -DUSE_CPUID=1)
+CFLAGS (GLOBAL -DUSE_JEMALLOC=0)
+CFLAGS (GLOBAL -DUSE_RAPIDJSON=1)
+
+IF (OS_DARWIN)
+    CFLAGS (GLOBAL -DOS_DARWIN)
+ELSEIF (OS_FREEBSD)
+    CFLAGS (GLOBAL -DOS_FREEBSD)
+ELSEIF (OS_LINUX)
+    CFLAGS (GLOBAL -DOS_LINUX)
+ENDIF ()
+
+PEERDIR(
+    contrib/libs/cctz/src
+    contrib/libs/cxxsupp/libcxx-filesystem
+    contrib/libs/poco/Net
+    contrib/libs/poco/Util
+    contrib/libs/fmt
+    contrib/restricted/boost
+    contrib/restricted/cityhash-1.0.2
+)
+
+SRCS(
+<? find . -name '*.cpp' | grep -v -F tests/ | grep -v -F Replxx | grep -v -F Readline | sed 's/^\.\//    /' | sort ?>
+)
+
+END()

contrib/cctz-cmake/CMakeLists.txt

@@ -28,13 +28,6 @@ if (USE_INTERNAL_CCTZ)
 
     if (OS_LINUX AND ARCH_AMD64)
 
-        find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy")
-        if (OBJCOPY_PATH)
-            message(STATUS "Using objcopy: ${OBJCOPY_PATH}.")
-        else ()
-            message(FATAL_ERROR "Cannot find objcopy.")
-        endif ()
-
         set (TIMEZONES
             Africa/Abidjan
             Africa/Accra

programs/server/CMakeLists.txt

@@ -3,6 +3,10 @@ set(CLICKHOUSE_SERVER_SOURCES
     Server.cpp
 )
 
+if (OS_LINUX AND ARCH_AMD64)
+    set (LINK_CONFIG_LIB INTERFACE "-Wl,--whole-archive $<TARGET_FILE:clickhouse_server_configs> -Wl,--no-whole-archive")
+endif ()
+
 set (CLICKHOUSE_SERVER_LINK
     PRIVATE
         clickhouse_aggregate_functions
@@ -16,18 +20,42 @@ set (CLICKHOUSE_SERVER_LINK
         clickhouse_table_functions
         string_utils
 
+    ${LINK_CONFIG_LIB}
+
     PUBLIC
         daemon
 )
 
 clickhouse_program_add(server)
 
-if (GLIBC_COMPATIBILITY)
-    set (GLIBC_MAX_REQUIRED 2.4 CACHE INTERNAL "")
-    # temporary disabled. to enable - change 'exit 0' to 'exit $a'
-    add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-server | perl -nE 'END {exit 0 if $a} ++$a, print if /\\x40GLIBC_(\\S+)/ and pack(q{C*}, split /\\./, \$1) gt pack q{C*}, split /\\./, q{${GLIBC_MAX_REQUIRED}}'")
-
-    #add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-server | grep '@GLIBC' | grep -oP 'GLIBC_[\\d\\.]+' | sort | uniq | sort --version-sort --reverse | perl -lnE 'warn($_), exit 1 if $_ gt q{GLIBC_${GLIBC_MAX_REQUIRED}}'") # old
-endif ()
-
 install(FILES config.xml users.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server COMPONENT clickhouse)
+
+# TODO We actually need this on Mac, FreeBSD and AArch64.
+if (OS_LINUX AND ARCH_AMD64)
+    # Embed default config files as a resource into the binary.
+    # This is needed for two purposes:
+    # 1. Allow to run the binary without download of any other files.
+    # 2. Allow to implement "sudo clickhouse install" tool.
+
+    foreach(CONFIG_FILE config users embedded)
+        set(CONFIG_OBJ ${CONFIG_FILE}.o)
+        set(CONFIG_OBJS ${CONFIG_OBJS} ${CONFIG_OBJ})
+
+        # https://stackoverflow.com/questions/14776463/compile-and-add-an-object-file-from-a-binary-with-cmake
+        add_custom_command(OUTPUT ${CONFIG_OBJ}
+            COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${OBJCOPY_PATH} -I binary -O elf64-x86-64 -B i386 ${CONFIG_FILE}.xml ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_OBJ}
+            COMMAND ${OBJCOPY_PATH} --rename-section .data=.rodata,alloc,load,readonly,data,contents
+                ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_OBJ} ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_OBJ})
+
+        set_source_files_properties(${CONFIG_OBJ} PROPERTIES EXTERNAL_OBJECT true GENERATED true)
+    endforeach(CONFIG_FILE)
+
+    add_library(clickhouse_server_configs STATIC ${CONFIG_OBJS})
+    set_target_properties(clickhouse_server_configs PROPERTIES LINKER_LANGUAGE C)
+
+    # whole-archive prevents symbols from being discarded for unknown reason
+    # CMake can shuffle each of target_link_libraries arguments with other
+    # libraries in linker command. To avoid this we hardcode whole-archive
+    # library into single string.
+    add_dependencies(clickhouse-server-lib clickhouse_server_configs)
+endif ()

programs/server/config.xml

@@ -154,7 +154,7 @@
     of the time, in which case a higher number of threads might be required.
     -->
 
-     <max_thread_pool_size>10000</max_thread_pool_size>
+    <max_thread_pool_size>10000</max_thread_pool_size>
 
     <!-- On memory constrained environments you may have to set this to value larger than 1.
       -->

programs/server/embedded.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!-- Config that is used when server is run without config file. -->
+<yandex>
+    <logger>
+        <level>trace</level>
+        <console>true</console>
+    </logger>
+
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <mysql_port>9004</mysql_port>
+
+    <path>./</path>
+
+    <uncompressed_cache_size>8589934592</uncompressed_cache_size>
+    <mark_cache_size>5368709120</mark_cache_size>
+    <mlock_executable>true</mlock_executable>
+
+    <users>
+        <default>
+            <password></password>
+
+            <networks incl="networks" replace="replace">
+                <ip>::/0</ip>
+            </networks>
+
+            <profile>default</profile>
+            <quota>default</quota>
+            <access_management>1</access_management>
+        </default>
+    </users>
+
+    <profiles>
+        <default/>
+    </profiles>
+
+    <quotas>
+        <default />
+    </quotas>
+</yandex>

src/Common/Config/ConfigProcessor.cpp

@@ -7,6 +7,7 @@
 #include <algorithm>
 #include <sstream>
 #include <functional>
+#include <filesystem>
 #include <Poco/DOM/Text.h>
 #include <Poco/DOM/Attr.h>
 #include <Poco/DOM/Comment.h>
@@ -14,6 +15,8 @@
 #include <Common/ZooKeeper/ZooKeeperNodeCache.h>
 #include <Common/ZooKeeper/KeeperException.h>
 #include <Common/StringUtils/StringUtils.h>
+#include <Common/Exception.h>
+#include <common/getResource.h>
 
 #define PREPROCESSED_SUFFIX "-preprocessed"
 
@@ -23,6 +26,11 @@ using namespace Poco::XML;
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int FILE_DOESNT_EXIST;
+}
+
 /// For cutting preprocessed path to this base
 static std::string main_config_path;
 
@@ -440,9 +448,27 @@ XMLDocumentPtr ConfigProcessor::processConfig(
     zkutil::ZooKeeperNodeCache * zk_node_cache,
     const zkutil::EventPtr & zk_changed_event)
 {
+    XMLDocumentPtr config;
     LOG_DEBUG(log, "Processing configuration file '{}'.", path);
 
-    XMLDocumentPtr config = dom_parser.parse(path);
+    if (std::filesystem::exists(path))
+    {
+        config = dom_parser.parse(path);
+    }
+    else
+    {
+        /// When we can use config embedded in binary.
+        if (path == "config.xml")
+        {
+            auto resource = getResource("embedded.xml");
+            if (resource.empty())
+                throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Configuration file {} doesn't exist and there is no embedded config", path);
+            LOG_DEBUG(log, "There is no file '{}', will use embedded config.", path);
+            config = dom_parser.parseMemory(resource.data(), resource.size());
+        }
+        else
+            throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Configuration file {} doesn't exist", path);
+    }
 
     std::vector<std::string> contributing_files;
     contributing_files.push_back(path);

src/Disks/ya.make

@@ -7,6 +7,8 @@ PEERDIR(
 
 SRCS(
     createVolume.cpp
+    DiskCacheWrapper.cpp
+    DiskDecorator.cpp
     DiskFactory.cpp
     DiskLocal.cpp
     DiskMemory.cpp