Add simple "clickhouse-install" tool

base/CMakeLists.txt

@@ -7,6 +7,7 @@ add_subdirectory (daemon)
 add_subdirectory (loggers)
 add_subdirectory (pcg-random)
 add_subdirectory (widechar_width)
+add_subdirectory (readpassphrase)
 
 if (USE_MYSQL)
     add_subdirectory (mysqlxx)

base/readpassphrase/CMakeLists.txt

@@ -5,3 +5,4 @@ add_library(readpassphrase readpassphrase.c)
 
 set_target_properties(readpassphrase PROPERTIES LINKER_LANGUAGE C)
 target_compile_options(readpassphrase PRIVATE -Wno-unused-result -Wno-reserved-id-macro)
+target_include_directories(readpassphrase PUBLIC .)

base/readpassphrase/readpassphrase.c

@@ -23,7 +23,9 @@
 
 /* OPENBSD ORIGINAL: lib/libc/gen/readpassphrase.c */
 
-#include "includes.h"
+#ifndef _PATH_TTY
+#define _PATH_TTY "/dev/tty"
+#endif
 
 #include <termios.h>
 #include <signal.h>

base/ya.make

@@ -4,4 +4,5 @@ RECURSE(
     loggers
     pcg-random
     widechar_width
+    readpassphrase
 )

programs/CMakeLists.txt

@@ -17,6 +17,7 @@ option (ENABLE_CLICKHOUSE_COPIER "Enable clickhouse-copier" ${ENABLE_CLICKHOUSE_
 option (ENABLE_CLICKHOUSE_FORMAT "Enable clickhouse-format" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_OBFUSCATOR "Enable clickhouse-obfuscator" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "Enable clickhouse-odbc-bridge" ${ENABLE_CLICKHOUSE_ALL})
+option (ENABLE_CLICKHOUSE_INSTALL "Enable clickhouse-install" ${ENABLE_CLICKHOUSE_ALL})
 
 if(NOT (MAKE_STATIC_LIBRARIES OR SPLIT_SHARED_LIBRARIES))
     set(CLICKHOUSE_ONE_SHARED 1)
@@ -84,6 +85,7 @@ add_subdirectory (compressor)
 add_subdirectory (copier)
 add_subdirectory (format)
 add_subdirectory (obfuscator)
+add_subdirectory (install)
 
 if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
     add_subdirectory (odbc-bridge)
@@ -142,6 +144,9 @@ else ()
     if (ENABLE_CLICKHOUSE_OBFUSCATOR)
         clickhouse_target_link_split_lib(clickhouse obfuscator)
     endif ()
+    if (ENABLE_CLICKHOUSE_INSTALL)
+        clickhouse_target_link_split_lib(clickhouse install)
+    endif ()
 
     set (CLICKHOUSE_BUNDLE)
     if (ENABLE_CLICKHOUSE_SERVER)
@@ -202,10 +207,6 @@ else ()
     endif()
 endif ()
 
-if (TARGET clickhouse-server AND TARGET copy-headers)
-    add_dependencies(clickhouse-server copy-headers)
-endif ()
-
 if (ENABLE_TESTS AND USE_GTEST)
     set (CLICKHOUSE_UNIT_TESTS_TARGETS unit_tests_libcommon unit_tests_dbms)
     add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_UNIT_TESTS_TARGETS})

programs/client/CMakeLists.txt

@@ -17,7 +17,6 @@ set (CLICKHOUSE_CLIENT_LINK
 )
 
 # Always use internal readpassphrase
-add_subdirectory(readpassphrase)
 list(APPEND CLICKHOUSE_CLIENT_LINK PRIVATE readpassphrase)
 
 clickhouse_program_add(client)

programs/client/ConnectionParameters.cpp

@@ -9,7 +9,7 @@
 #include <Common/Exception.h>
 #include <common/setTerminalEcho.h>
 #include <ext/scope_guard.h>
-#include "readpassphrase/readpassphrase.h"
+#include <readpassphrase.h>
 
 namespace DB
 {

programs/client/readpassphrase/includes.h

@@ -1,9 +0,0 @@
-#pragma once
-
-/* #undef HAVE_READPASSPHRASE */
-
-#if !defined(HAVE_READPASSPHRASE)
-#    ifndef _PATH_TTY
-#        define _PATH_TTY "/dev/tty"
-#    endif
-#endif

programs/config_tools.h.in

@@ -12,4 +12,5 @@
 #cmakedefine01 ENABLE_CLICKHOUSE_COMPRESSOR
 #cmakedefine01 ENABLE_CLICKHOUSE_FORMAT
 #cmakedefine01 ENABLE_CLICKHOUSE_OBFUSCATOR
+#cmakedefine01 ENABLE_CLICKHOUSE_INSTALL
 #cmakedefine01 ENABLE_CLICKHOUSE_ODBC_BRIDGE

programs/install/CMakeLists.txt

@@ -0,0 +1,12 @@
+set (CLICKHOUSE_INSTALL_SOURCES Install.cpp)
+
+set (CLICKHOUSE_INSTALL_LINK
+    PRIVATE
+        boost::program_options
+        clickhouse_common_io
+        dbms
+        readpassphrase
+)
+
+clickhouse_program_add(install)
+

programs/install/Install.cpp

@@ -0,0 +1,506 @@
+#include <iostream>
+#include <filesystem>
+#include <boost/program_options.hpp>
+
+#include <sys/stat.h>
+
+#if defined(__linux__)
+    #include <syscall.h>
+    #include <linux/capability.h>
+#endif
+
+#include <Common/Exception.h>
+#include <Common/ShellCommand.h>
+#include <Common/formatReadable.h>
+#include <Common/Config/ConfigProcessor.h>
+#include <Common/OpenSSLHelpers.h>
+#include <Common/hex.h>
+#include <common/getResource.h>
+#include <IO/ReadBufferFromFileDescriptor.h>
+#include <IO/WriteBufferFromFileDescriptor.h>
+#include <IO/ReadBufferFromFile.h>
+#include <IO/WriteBufferFromFile.h>
+#include <IO/ReadBufferFromMemory.h>
+#include <IO/copyData.h>
+#include <IO/Operators.h>
+#include <readpassphrase.h>
+
+#include <Poco/Util/XMLConfiguration.h>
+
+
+/** This tool can be used to install ClickHouse without a deb/rpm/tgz package, having only "clickhouse" binary.
+  * The following steps are performed:
+  *
+  * - copying the binary to binary directory (/usr/bin).
+  * - creation of symlinks for tools.
+  * - creation of clickhouse user and group.
+  * - creation of config directory (/etc/clickhouse-server).
+  * - creation of default configuration files.
+  * - creation of a directory for logs (/var/log/clickhouse-server).
+  * - creation of a data directory if not exists.
+  * - setting a password for default user.
+  * - choose an option to listen connections.
+  * - changing the ownership and mode of the directories.
+  * - setting capabilities for binary.
+  * - setting ulimits for the user.
+  *
+  * It does not install clickhouse-odbc-bridge.
+  */
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int FILE_DOESNT_EXIST;
+    extern const int CANNOT_OPEN_FILE;
+    extern const int SYSTEM_ERROR;
+    extern const int NOT_ENOUGH_SPACE;
+}
+
+void executeScript(const std::string & command)
+{
+    auto sh = ShellCommand::execute(command);
+    WriteBufferFromFileDescriptor stdout(STDOUT_FILENO);
+    WriteBufferFromFileDescriptor stderr(STDERR_FILENO);
+    copyData(sh->out, stdout);
+    copyData(sh->err, stderr);
+    sh->tryWait();
+}
+
+bool ask(std::string question)
+{
+    while (true)
+    {
+        std::string answer;
+        std::cout << question;
+        std::getline(std::cin, answer);
+        if (!std::cin.good())
+            return false;
+
+        if (answer.empty() || answer == "n" || answer == "N")
+            return false;
+        if (answer == "y" || answer == "Y")
+            return true;
+    }
+}
+
+}
+
+
+int mainEntryClickHouseInstall(int argc, char ** argv)
+{
+    using namespace DB;
+    namespace po = boost::program_options;
+    namespace fs = std::filesystem;
+
+    po::options_description desc;
+    desc.add_options()
+        ("help,h", "produce help message")
+        ("prefix", po::value<std::string>()->default_value(""), "prefix for all paths")
+        ("binary-path", po::value<std::string>()->default_value("/usr/bin"), "where to install binaries")
+        ("config-path", po::value<std::string>()->default_value("/etc/clickhouse-server"), "where to install configs")
+        ("log-path", po::value<std::string>()->default_value("/var/log/clickhouse-server"), "where to create log directory")
+        ("data-path", po::value<std::string>()->default_value("/var/lib/clickhouse"), "directory for data")
+        ("user", po::value<std::string>()->default_value("clickhouse"), "clickhouse user to create")
+        ("group", po::value<std::string>()->default_value("clickhouse"), "clickhouse group to create")
+    ;
+
+    po::variables_map options;
+    po::store(po::parse_command_line(argc, argv, desc), options);
+
+    if (options.count("help"))
+    {
+        std::cout << "Usage: "
+            << (getuid() == 0 ? "" : "sudo ")
+            << argv[0]
+            << " install [options]\n";
+        std::cout << desc << '\n';
+        return 1;
+    }
+
+    try
+    {
+        /// We need to copy binary to the binary directory.
+        /// The binary is currently run. We need to obtain its path from procfs.
+
+        fs::path binary_self_path = "/proc/self/exe";
+        if (!fs::exists(binary_self_path))
+            throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Cannot obtain path to the binary from {}, file doesn't exist",
+                            binary_self_path.string());
+
+        /// Copy binary to the destination directory.
+
+        size_t binary_size = fs::file_size(binary_self_path);
+
+        fs::path prefix = fs::path(options["prefix"].as<std::string>());
+        fs::path bin_dir = prefix / fs::path(options["binary-path"].as<std::string>());
+
+        size_t available_space = fs::space(bin_dir).available;
+        if (available_space < binary_size)
+            throw Exception(ErrorCodes::NOT_ENOUGH_SPACE, "Not enough space for clickhouse binary in {}, required {}, available {}.",
+                bin_dir.string(), ReadableSize(binary_size), ReadableSize(available_space));
+
+        fs::path main_bin_path = bin_dir / "clickhouse";
+        fs::path main_bin_tmp_path = bin_dir / "clickhouse.new";
+        fs::path main_bin_old_path = bin_dir / "clickhouse.old";
+
+        fmt::print("Copying ClickHouse binary to {}\n", main_bin_tmp_path.string());
+
+        try
+        {
+            ReadBufferFromFile in(binary_self_path.string());
+            WriteBufferFromFile out(main_bin_tmp_path.string());
+            copyData(in, out);
+            out.sync();
+
+            if (0 != fchmod(out.getFD(), S_IRUSR | S_IRGRP | S_IROTH | S_IXUSR | S_IXGRP | S_IXOTH))
+                throwFromErrno(fmt::format("Cannot chmod {}", main_bin_tmp_path.string()), ErrorCodes::SYSTEM_ERROR);
+
+            out.finalize();
+        }
+        catch (const Exception & e)
+        {
+            if (e.code() == ErrorCodes::CANNOT_OPEN_FILE && geteuid() != 0)
+                std::cerr << "Install must be run as root: sudo ./clickhouse install";
+            throw;
+        }
+
+        if (fs::exists(main_bin_path))
+        {
+            fmt::print("{} already exists, will rename existing binary to {} and put the new binary in place\n",
+                       main_bin_path.string(), main_bin_old_path.string());
+
+            /// There is file exchange operation in Linux but it's not portable.
+            fs::rename(main_bin_path, main_bin_old_path);
+        }
+
+        fmt::print("Renaming {} to {}.\n", main_bin_tmp_path.string(), main_bin_path.string());
+        fs::rename(main_bin_tmp_path, main_bin_path);
+
+        /// Create symlinks.
+
+        std::initializer_list<const char *> tools
+        {
+            "clickhouse-server",
+            "clickhouse-client",
+            "clickhouse-local",
+            "clickhouse-benchmark",
+            "clickhouse-copier",
+            "clickhouse-obfuscator",
+            "clickhouse-compressor",
+            "clickhouse-format",
+            "clickhouse-extract-from-config"
+        };
+
+        for (const auto & tool : tools)
+        {
+            bool need_to_create = true;
+            fs::path symlink_path = bin_dir / tool;
+
+            if (fs::exists(symlink_path))
+            {
+                bool is_symlink = fs::is_symlink(symlink_path);
+                fs::path points_to;
+                if (is_symlink)
+                    points_to = fs::absolute(fs::read_symlink(symlink_path));
+
+                if (is_symlink && points_to == main_bin_path)
+                {
+                    need_to_create = false;
+                }
+                else
+                {
+                    if (!is_symlink)
+                    {
+                        fs::path rename_path = symlink_path.replace_extension(".old");
+                        fmt::print("File {} already exists but it's not a symlink. Will rename to {}.\n",
+                                   symlink_path.string(), rename_path.string());
+                        fs::rename(symlink_path, rename_path);
+                    }
+                    else if (points_to != main_bin_path)
+                    {
+                        fmt::print("Symlink {} already exists but it points to {}. Will replace the old symlink to {}.\n",
+                                   symlink_path.string(), points_to.string(), main_bin_path.string());
+                        fs::remove(symlink_path);
+                    }
+                }
+            }
+
+            if (need_to_create)
+            {
+                fmt::print("Creating symlink {} to {}.\n", symlink_path.string(), main_bin_path.string());
+                fs::create_symlink(main_bin_path, symlink_path);
+            }
+        }
+
+        /// Creation of clickhouse user and group.
+
+        std::string user = options["user"].as<std::string>();
+        std::string group = options["group"].as<std::string>();
+
+        if (!group.empty())
+        {
+            {
+                fmt::print("Creating clickhouse group if it does not exist.\n");
+                std::string command = fmt::format("groupadd -r {}", group);
+                fmt::print(" {}\n", command);
+                executeScript(command);
+            }
+        }
+        else
+            fmt::print("Will not create clickhouse group");
+
+        if (!user.empty())
+        {
+            fmt::print("Creating clickhouse user if it does not exist.\n");
+            std::string command = group.empty()
+                ? fmt::format("useradd --system --shell /bin/false --home-dir /nonexistent --user-group {}", user)
+                : fmt::format("useradd --system --shell /bin/false --home-dir /nonexistent -g {} {}", group, user);
+            fmt::print(" {}\n", command);
+            executeScript(command);
+
+            /// Setting ulimits.
+            try
+            {
+                fs::path ulimits_dir = "/etc/security/limits.d";
+                fs::path ulimits_file = ulimits_dir / fmt::format("{}.conf", user);
+                fmt::print("Will set ulimits for {} user in {}.\n", user, ulimits_file.string());
+                std::string ulimits_content = fmt::format(
+                    "{0}\tsoft\tnofile\t262144\n"
+                    "{0}\thard\tnofile\t262144\n", user);
+
+                fs::create_directories(ulimits_dir);
+
+                WriteBufferFromFile out(ulimits_file.string());
+                out.write(ulimits_content.data(), ulimits_content.size());
+                out.sync();
+                out.finalize();
+            }
+            catch (...)
+            {
+                std::cerr << "Cannot set ulimits: " << getCurrentExceptionMessage(false) << "\n";
+            }
+
+            /// TODO Set ulimits on Mac OS X
+        }
+        else
+            fmt::print("Will not create clickhouse user.\n");
+
+        /// Creating configuration files and directories.
+
+        fs::path config_dir = prefix / options["config-path"].as<std::string>();
+
+        if (!fs::exists(config_dir))
+        {
+            fmt::print("Creating config directory {}.\n", config_dir.string());
+            fs::create_directories(config_dir);
+        }
+
+        fs::path main_config_file = config_dir / "config.xml";
+        fs::path users_config_file = config_dir / "users.xml";
+        fs::path config_d = config_dir / "config.d";
+        fs::path users_d = config_dir / "users.d";
+
+        std::string log_path = prefix / options["log-path"].as<std::string>();
+        std::string data_path = prefix / options["data-path"].as<std::string>();
+
+        bool has_password_for_default_user = false;
+
+        if (!fs::exists(main_config_file))
+        {
+            fmt::print("Creating config directory {} that is used for tweaks of main server configuration.\n", config_d.string());
+            fs::create_directory(config_d);
+
+            fmt::print("Creating config directory {} that is used for tweaks of users configuration.\n", users_d.string());
+            fs::create_directory(users_d);
+
+            std::string_view main_config_content = getResource("config.xml");
+            if (main_config_content.empty())
+            {
+                fmt::print("There is no default config.xml, you have to download it and place to {}.\n", main_config_file.string());
+            }
+            else
+            {
+                WriteBufferFromFile out(main_config_file.string());
+                out.write(main_config_content.data(), main_config_content.size());
+                out.sync();
+                out.finalize();
+            }
+
+            std::string_view users_config_content = getResource("users.xml");
+            if (users_config_content.empty())
+            {
+                fmt::print("There is no default users.xml, you have to download it and place to {}.\n", users_config_file.string());
+            }
+            else
+            {
+                WriteBufferFromFile out(users_config_file.string());
+                out.write(users_config_content.data(), users_config_content.size());
+                out.sync();
+                out.finalize();
+            }
+
+            /// TODO chmod configs?
+        }
+        else
+        {
+            {
+                fmt::print("Config file {} already exists, will keep it and extract path info from it.\n", main_config_file.string());
+
+                ConfigProcessor processor(main_config_file.string(), /* throw_on_bad_incl = */ false, /* log_to_console = */ false);
+                ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(processor.processConfig()));
+
+                if (configuration->has("path"))
+                {
+                    data_path = configuration->getString("path");
+                    fmt::print("{} has {} as data path.\n", main_config_file.string(), data_path);
+                }
+
+                if (configuration->has("logger.log"))
+                {
+                    log_path = fs::path(configuration->getString("logger.log")).remove_filename();
+                    fmt::print("{} has {} as log path.\n", main_config_file.string(), log_path);
+                }
+            }
+
+            /// Check if password for default user already specified.
+
+            if (fs::exists(users_config_file))
+            {
+                ConfigProcessor processor(users_config_file.string(), /* throw_on_bad_incl = */ false, /* log_to_console = */ false);
+                ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(processor.processConfig()));
+
+                if (!configuration->getString("users.default.password", "").empty()
+                    || configuration->getString("users.default.password_sha256_hex", "").empty()
+                    || configuration->getString("users.default.password_double_sha1_hex", "").empty())
+                {
+                    has_password_for_default_user = true;
+                }
+            }
+        }
+
+        /// Create directories for data and log.
+
+        if (fs::exists(log_path))
+        {
+            fmt::print("Log directory {} already exists.\n", log_path);
+        }
+        else
+        {
+            fmt::print("Creating log directory {}.\n", log_path);
+            fs::create_directories(log_path);
+        }
+
+        if (fs::exists(data_path))
+        {
+            fmt::print("Data directory {} already exists.\n", data_path);
+        }
+        else
+        {
+            fmt::print("Creating data directory {}.\n", data_path);
+            fs::create_directories(data_path);
+        }
+
+        /// TODO chmod dirs
+
+        /// Set up password for default user.
+
+        bool stdin_is_a_tty = isatty(STDIN_FILENO);
+        bool stdout_is_a_tty = isatty(STDOUT_FILENO);
+        bool is_interactive = stdin_is_a_tty && stdout_is_a_tty;
+
+        if (has_password_for_default_user)
+        {
+            fmt::print("Password for default user is already specified. To remind or reset, see {} and {}.\n",
+                       users_config_file.string(), users_d.string());
+        }
+        else if (!is_interactive)
+        {
+            fmt::print("Password for default user is empty string. See {} and {} to change it.\n",
+                       users_config_file.string(), users_d.string());
+        }
+        else
+        {
+            char buf[1000] = {};
+            std::string password;
+            if (auto * result = readpassphrase("Enter password for default user: ", buf, sizeof(buf), 0))
+                password = result;
+
+            if (!password.empty())
+            {
+                std::string password_file = users_d / "default-password.xml";
+                WriteBufferFromFile out(password_file);
+#if USE_SSL
+                std::vector<uint8_t> hash;
+                hash.resize(32);
+                encodeSHA256(password, hash.data());
+                std::string hash_hex;
+                hash_hex.resize(64);
+                for (size_t i = 0; i < 32; ++i)
+                    writeHexByteLowercase(hash[i], &hash_hex[2 * i]);
+                out << "<yandex>\n"
+                    "    <users>\n"
+                    "        <default>\n"
+                    "            <password_sha256_hex>" << hash_hex << "</password_sha256_hex>\n"
+                    "        </default>\n"
+                    "    </users>\n"
+                    "</yandex>\n";
+                out.sync();
+                out.finalize();
+                fmt::print("Password for default user is saved in file {}.\n", password_file);
+#else
+                out << "<yandex>\n"
+                    "    <users>\n"
+                    "        <default>\n"
+                    "            <password><![CDATA[" << password << "]]></password>\n"
+                    "        </default>\n"
+                    "    </users>\n"
+                    "</yandex>\n";
+                out.sync();
+                out.finalize();
+                fmt::print("Password for default user is saved in plaintext in file {}.\n", password_file);
+#endif
+                has_password_for_default_user = true;
+            }
+            else
+                fmt::print("Password for default user is empty string. See {} and {} to change it.\n",
+                           users_config_file.string(), users_d.string());
+        }
+
+        /// Set capabilities for the binary.
+
+#if defined(__linux__)
+        fmt::print("Setting capabilities for clickhouse binary. This is optional.\n");
+        std::string command = fmt::format("command setcap && setcap 'cap_net_admin,cap_ipc_lock,cap_sys_nice+ep' {}", main_bin_path.string());
+        fmt::print(" {}\n", command);
+        executeScript(command);
+#endif
+
+        /// If password was set, ask for open for connections.
+        if (is_interactive && has_password_for_default_user)
+        {
+            if (ask("Allow server to accept connections from the network (default is localhost only), [y/N]: "))
+            {
+                std::string listen_file = config_d / "listen.xml";
+                WriteBufferFromFile out(listen_file);
+                out << "<yandex>\n"
+                    "    <listen_host>::</listen_host>\n"
+                    "</yandex>\n";
+                out.sync();
+                out.finalize();
+                fmt::print("The choice is saved in file {}.\n", listen_file);
+            }
+        }
+
+        fmt::print("\nClickHouse has been successfully installed.\n");
+    }
+    catch (...)
+    {
+        std::cerr << getCurrentExceptionMessage(false);
+        return getCurrentExceptionCode();
+    }
+
+    return 0;
+}
+

programs/main.cpp

@@ -46,6 +46,9 @@ int mainEntryClickHouseClusterCopier(int argc, char ** argv);
 #if ENABLE_CLICKHOUSE_OBFUSCATOR
 int mainEntryClickHouseObfuscator(int argc, char ** argv);
 #endif
+#if ENABLE_CLICKHOUSE_INSTALL
+int mainEntryClickHouseInstall(int argc, char ** argv);
+#endif
 
 
 namespace
@@ -84,6 +87,9 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
 #if ENABLE_CLICKHOUSE_OBFUSCATOR
     {"obfuscator", mainEntryClickHouseObfuscator},
 #endif
+#if ENABLE_CLICKHOUSE_INSTALL
+    {"install", mainEntryClickHouseInstall},
+#endif
 };
 
 

programs/ya.make

@@ -9,7 +9,7 @@ CFLAGS(
 PEERDIR(
     clickhouse/base/daemon
     clickhouse/base/loggers
-    clickhouse/programs/client/readpassphrase
+    clickhouse/base/readpassphrase
     clickhouse/src
 )