Use logging in clickhouse-local. Use config options in command line in clickhouse-client (#5540)

* Try fix macos server run

* Doc macos build
 # Please enter the commit message for your changes. Lines starting

* CLICKHOUSE-3957 start wip

* tests wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* fix

* fix

* Making logger for clickhouse-local

* fixes

* wip

* wip

* wip

* wip

* clean

*      cf

* wip

* fix

* Update CMakeLists.txt

* Update argsToConfig.h

* Update argsToConfig.cpp

* Update ExtendedLogChannel.h

* Update OwnPatternFormatter.cpp

dbms/programs/client/Client.cpp

@@ -65,6 +65,7 @@
 #include <AggregateFunctions/registerAggregateFunctions.h>
 #include <Common/Config/configReadClient.h>
 #include <Storages/ColumnsDescription.h>
+#include <common/argsToConfig.h>
 
 #if USE_READLINE
 #include "Suggest.h"
@@ -1549,7 +1550,7 @@ public:
           *   where possible args are file, name, format, structure, types.
           * Split these groups before processing.
           */
-        using Arguments = std::vector<const char *>;
+        using Arguments = std::vector<std::string>;
 
         Arguments common_arguments{""};        /// 0th argument is ignored.
         std::vector<Arguments> external_tables_arguments;
@@ -1671,8 +1672,7 @@ public:
             ("types", po::value<std::string>(), "types")
         ;
         /// Parse main commandline options.
-        po::parsed_options parsed = po::command_line_parser(
-            common_arguments.size(), common_arguments.data()).options(main_description).run();
+        po::parsed_options parsed = po::command_line_parser(common_arguments).options(main_description).run();
         po::variables_map options;
         po::store(parsed, options);
         po::notify(options);
@@ -1705,8 +1705,7 @@ public:
         for (size_t i = 0; i < external_tables_arguments.size(); ++i)
         {
             /// Parse commandline options related to external tables.
-            po::parsed_options parsed_tables = po::command_line_parser(
-                external_tables_arguments[i].size(), external_tables_arguments[i].data()).options(external_description).run();
+            po::parsed_options parsed_tables = po::command_line_parser(external_tables_arguments[i]).options(external_description).run();
             po::variables_map external_options;
             po::store(parsed_tables, external_options);
 
@@ -1802,6 +1801,9 @@ public:
         }
         if (options.count("suggestion_limit"))
             config().setInt("suggestion_limit", options["suggestion_limit"].as<int>());
+
+        argsToConfig(common_arguments, config(), 100);
+
     }
 };
 

dbms/programs/copier/ClusterCopier.cpp

@@ -16,7 +16,6 @@
 #include <pcg_random.hpp>
 #include <common/logger_useful.h>
 #include <Common/ThreadPool.h>
-#include <daemon/OwnPatternFormatter.h>
 #include <Common/Exception.h>
 #include <Common/ZooKeeper/ZooKeeper.h>
 #include <Common/ZooKeeper/KeeperException.h>

dbms/programs/local/CMakeLists.txt

@@ -6,4 +6,4 @@ clickhouse_program_add(local)
 
 if(NOT CLICKHOUSE_ONE_SHARED)
     target_link_libraries(clickhouse-local-lib PRIVATE clickhouse-server-lib)
-endif ()
+endif()

dbms/programs/local/LocalServer.cpp

@@ -59,11 +59,29 @@ void LocalServer::initialize(Poco::Util::Application & self)
 {
     Poco::Util::Application::initialize(self);
 
-    // Turn off server logging to stderr
-    if (!config().has("verbose"))
+    /// Load config files if exists
+    if (config().has("config-file") || Poco::File("config.xml").exists())
     {
-        Poco::Logger::root().setLevel("none");
-        Poco::Logger::root().setChannel(Poco::AutoPtr<Poco::NullChannel>(new Poco::NullChannel()));
+        const auto config_path = config().getString("config-file", "config.xml");
+        ConfigProcessor config_processor(config_path, false, true);
+        config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString());
+        auto loaded_config = config_processor.loadConfig();
+        config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", "."));
+        config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
+    }
+
+    if (config().has("logger") || config().has("logger.level") || config().has("logger.log"))
+    {
+        buildLoggers(config(), logger());
+    }
+    else
+    {
+        // Turn off server logging to stderr
+        if (!config().has("verbose"))
+        {
+            Poco::Logger::root().setLevel("none");
+            Poco::Logger::root().setChannel(Poco::AutoPtr<Poco::NullChannel>(new Poco::NullChannel()));
+        }
     }
 }
 
@@ -110,16 +128,6 @@ try
         return Application::EXIT_OK;
     }
 
-    /// Load config files if exists
-    if (config().has("config-file") || Poco::File("config.xml").exists())
-    {
-        const auto config_path = config().getString("config-file", "config.xml");
-        ConfigProcessor config_processor(config_path, false, true);
-        config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString());
-        auto loaded_config = config_processor.loadConfig();
-        config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", DBMS_DEFAULT_PATH));
-        config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
-    }
 
     context = std::make_unique<Context>(Context::createGlobal());
     context->setGlobalContext(*context);
@@ -428,6 +436,8 @@ void LocalServer::init(int argc, char ** argv)
         ("stacktrace", "print stack traces of exceptions")
         ("echo", "print query before execution")
         ("verbose", "print query and other debugging info")
+        ("logger.log", po::value<std::string>(), "Log file name")
+        ("logger.level", po::value<std::string>(), "Log level")
         ("ignore-error", "do not stop processing if a query failed")
         ("version,V", "print version information and exit")
         ;
@@ -481,6 +491,10 @@ void LocalServer::init(int argc, char ** argv)
         config().setBool("echo", true);
     if (options.count("verbose"))
         config().setBool("verbose", true);
+    if (options.count("logger.log"))
+        config().setString("logger.log", options["logger.log"].as<std::string>());
+    if (options.count("logger.level"))
+        config().setString("logger.level", options["logger.level"].as<std::string>());
     if (options.count("ignore-error"))
         config().setBool("ignore-error", true);
 }

dbms/programs/local/LocalServer.h

@@ -3,6 +3,7 @@
 #include <Core/Settings.h>
 #include <Poco/Util/Application.h>
 #include <memory>
+#include <loggers/Loggers.h>
 
 
 namespace DB
@@ -13,7 +14,7 @@ class Context;
 /// Lightweight Application for clickhouse-local
 /// No networking, no extra configs and working directories, no pid and status files, no dictionaries, no logging.
 /// Quiet mode by default
-class LocalServer : public Poco::Util::Application
+class LocalServer : public Poco::Util::Application, public Loggers
 {
 public:
     LocalServer();

dbms/programs/odbc-bridge/ODBCBridge.cpp

@@ -123,7 +123,7 @@ void ODBCBridge::initialize(Application & self)
 
     config().setString("logger", "ODBCBridge");
 
-    buildLoggers(config());
+    buildLoggers(config(), logger());
     log = &logger();
     hostname = config().getString("listen-host", "localhost");
     port = config().getUInt("http-port");

dbms/programs/server/Server.cpp

@@ -405,7 +405,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
         main_config_zk_changed_event,
         [&](ConfigurationPtr config)
         {
-            buildLoggers(*config);
+            buildLoggers(*config, logger());
             global_context->setClustersConfig(config);
             global_context->setMacros(std::make_unique<Macros>(*config, "macros"));
         },

dbms/programs/server/TCPHandler.cpp

@@ -1,8 +1,6 @@
 #include <iomanip>
 #include <ext/scope_guard.h>
 #include <Poco/Net/NetException.h>
-#include <daemon/OwnSplitChannel.h>
-
 #include <Common/ClickHouseRevision.h>
 #include <Common/CurrentThread.h>
 #include <Common/Stopwatch.h>

dbms/tests/queries/shell_config.sh

@@ -22,7 +22,7 @@ export CLICKHOUSE_CONFIG_CLIENT=${CLICKHOUSE_CONFIG_CLIENT:="/etc/clickhouse-cli
 [ -x "${CLICKHOUSE_BINARY}" ] && CLICKHOUSE_EXTRACT_CONFIG=${CLICKHOUSE_EXTRACT_CONFIG:="$CLICKHOUSE_BINARY extract-from-config --config=$CLICKHOUSE_CONFIG"}
 export CLICKHOUSE_EXTRACT_CONFIG=${CLICKHOUSE_EXTRACT_CONFIG:="$CLICKHOUSE_BINARY-extract-from-config --config=$CLICKHOUSE_CONFIG"}
 
-[ -x "${CLICKHOUSE_BINARY}-format" ] && CLICKHOUSE_FORMAT=${CLICKHOUSE_FORMAT=:="$CLICKHOUSE_BINARY-format"}
+[ -x "${CLICKHOUSE_BINARY}-format" ] && CLICKHOUSE_FORMAT=${CLICKHOUSE_FORMAT:="$CLICKHOUSE_BINARY-format"}
 [ -x "${CLICKHOUSE_BINARY}" ] && CLICKHOUSE_FORMAT=${CLICKHOUSE_FORMAT:="$CLICKHOUSE_BINARY format"}
 export CLICKHOUSE_FORMAT=${CLICKHOUSE_FORMAT:="$CLICKHOUSE_BINARY-format"}
 

libs/CMakeLists.txt

@@ -8,6 +8,7 @@ if (USE_DEBUG_HELPERS)
 endif ()
 
 add_subdirectory (libcommon)
+add_subdirectory (libloggers)
 add_subdirectory (libdaemon)
 
 if (USE_INTERNAL_MEMCPY)

libs/libcommon/CMakeLists.txt

@@ -21,6 +21,7 @@ add_library(common
     src/demangle.cpp
     src/setTerminalEcho.cpp
     src/getThreadNumber.cpp
+    src/argsToConfig.cpp
 
     include/common/Types.h
     include/common/DayNum.h
@@ -107,6 +108,7 @@ endif()
 
 target_link_libraries (common
         PUBLIC
+    ${Poco_Util_LIBRARY}
     ${Poco_Foundation_LIBRARY}
     ${CITYHASH_LIBRARIES}
         PRIVATE

libs/libcommon/include/common/argsToConfig.h

@@ -0,0 +1,10 @@
+#pragma once
+#include <Poco/Util/Application.h>
+
+namespace Poco::Util
+{
+class LayeredConfiguration;
+}
+
+/// Import extra command line arguments to configuration. These are command line arguments after --.
+void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::LayeredConfiguration & config, int priority);

libs/libcommon/src/argsToConfig.cpp

@@ -0,0 +1,67 @@
+#include <common/argsToConfig.h>
+
+#include <Poco/Util/Application.h>
+#include <Poco/Util/LayeredConfiguration.h>
+#include <Poco/Util/MapConfiguration.h>
+
+
+void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::LayeredConfiguration & config, int priority)
+{
+    /// Parsing all args and converting to config layer
+    /// Test: -- --1=1 --1=2 --3 5 7 8 -9 10 -11=12 14= 15== --16==17 --=18 --19= --20 21 22 --23 --24 25 --26 -27 28 ---29=30 -- ----31 32 --33 3-4
+    Poco::AutoPtr<Poco::Util::MapConfiguration> map_config = new Poco::Util::MapConfiguration;
+    std::string key;
+    for (auto & arg : argv)
+    {
+        auto key_start = arg.find_first_not_of('-');
+        auto pos_minus = arg.find('-');
+        auto pos_eq = arg.find('=');
+
+        // old saved '--key', will set to some true value "1"
+        if (!key.empty() && pos_minus != std::string::npos && pos_minus < key_start)
+        {
+            map_config->setString(key, "1");
+            key = "";
+        }
+
+        if (pos_eq == std::string::npos)
+        {
+            if (!key.empty())
+            {
+                if (pos_minus == std::string::npos || pos_minus > key_start)
+                {
+                    map_config->setString(key, arg);
+                }
+                key = "";
+            }
+            if (pos_minus != std::string::npos && key_start != std::string::npos && pos_minus < key_start)
+                key = arg.substr(key_start);
+            continue;
+        }
+        else
+        {
+            key = "";
+        }
+
+        if (key_start == std::string::npos)
+            continue;
+
+        if (pos_minus > key_start)
+            continue;
+
+        key = arg.substr(key_start, pos_eq - key_start);
+        if (key.empty())
+            continue;
+        std::string value;
+        if (arg.size() > pos_eq)
+            value = arg.substr(pos_eq + 1);
+
+        map_config->setString(key, value);
+        key = "";
+    }
+
+    Poco::Util::MapConfiguration::Keys keys;
+    map_config->keys(keys);
+
+    config.add(map_config, priority);
+}

libs/libdaemon/CMakeLists.txt

@@ -1,17 +1,9 @@
 add_library (daemon
     src/BaseDaemon.cpp
     src/GraphiteWriter.cpp
-    src/ExtendedLogChannel.cpp
-    src/OwnPatternFormatter.cpp
-    src/OwnFormattingChannel.cpp
-    src/OwnSplitChannel.cpp
 
     include/daemon/BaseDaemon.h
     include/daemon/GraphiteWriter.h
-    include/daemon/ExtendedLogChannel.h
-    include/daemon/OwnPatternFormatter.h
-    include/daemon/OwnFormattingChannel.h
-    include/daemon/OwnSplitChannel.h
 )
 
 if (USE_UNWIND)
@@ -22,4 +14,4 @@ endif ()
 
 target_include_directories (daemon PUBLIC include)
 
-target_link_libraries (daemon PRIVATE clickhouse_common_io clickhouse_common_config common ${Poco_Net_LIBRARY} ${Poco_Util_LIBRARY} ${EXECINFO_LIBRARIES})
+target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickhouse_common_config common ${Poco_Net_LIBRARY} ${Poco_Util_LIBRARY} ${EXECINFO_LIBRARIES})

libs/libdaemon/include/daemon/BaseDaemon.h

@@ -16,13 +16,12 @@
 #include <Poco/Util/Application.h>
 #include <Poco/Util/ServerApplication.h>
 #include <Poco/Net/SocketAddress.h>
-#include <Poco/FileChannel.h>
-#include <Poco/SyslogChannel.h>
 #include <Poco/Version.h>
 #include <common/Types.h>
 #include <common/logger_useful.h>
 #include <daemon/GraphiteWriter.h>
 #include <Common/Config/ConfigProcessor.h>
+#include <loggers/Loggers.h>
 
 namespace Poco { class TaskManager; }
 
@@ -40,7 +39,7 @@ namespace Poco { class TaskManager; }
 ///
 /// You can configure different log options for different loggers used inside program
 ///  by providing subsections to "logger" in configuration file.
-class BaseDaemon : public Poco::Util::ServerApplication
+class BaseDaemon : public Poco::Util::ServerApplication, public Loggers
 {
     friend class SignalListener;
 
@@ -56,9 +55,6 @@ public:
     /// Читает конфигурацию
     void reloadConfiguration();
 
-    /// Строит необходимые логгеры
-    void buildLoggers(Poco::Util::AbstractConfiguration & config);
-
     /// Определяет параметр командной строки
     void defineOptions(Poco::Util::OptionSet & _options) override;
 
@@ -92,9 +88,6 @@ public:
     /// Разбудить
     void wakeup();
 
-    /// Закрыть файлы с логами. При следующей записи, будут созданы новые файлы.
-    void closeLogs();
-
     /// В Graphite компоненты пути(папки) разделяются точкой.
     /// У нас принят путь формата root_path.hostname_yandex_ru.key
     /// root_path по умолчанию one_min
@@ -130,11 +123,6 @@ public:
         return nullptr;
     }
 
-    std::optional<size_t> getLayer() const
-    {
-        return layer;    /// layer выставляется в классе-наследнике BaseDaemonApplication.
-    }
-
     /// close all process FDs except
     /// 0-2 -- stdin, stdout, stderr
     /// also doesn't close global internal pipes for signal handling
@@ -209,15 +197,8 @@ protected:
     Poco::Thread signal_listener_thread;
     std::unique_ptr<Poco::Runnable> signal_listener;
 
-    /// Файлы с логами.
-    Poco::AutoPtr<Poco::FileChannel> log_file;
-    Poco::AutoPtr<Poco::FileChannel> error_log_file;
-    Poco::AutoPtr<Poco::Channel> syslog_channel;
-
     std::map<std::string, std::unique_ptr<GraphiteWriter>> graphite_writers;
 
-    std::optional<size_t> layer;
-
     std::mutex signal_handler_mutex;
     std::condition_variable signal_event;
     std::atomic_size_t terminate_signals_counter{0};
@@ -229,9 +210,6 @@ protected:
 
 private:
 
-    /// Previous value of logger element in config. It is used to reinitialize loggers whenever the value changed.
-    std::string config_logger;
-
     /// Check SSE and others instructions availability
     /// Calls exit on fail
     void checkRequiredInstructions();

libs/libdaemon/src/BaseDaemon.cpp

@@ -1,8 +1,6 @@
 #include <daemon/BaseDaemon.h>
-#include <daemon/OwnFormattingChannel.h>
-#include <daemon/OwnPatternFormatter.h>
+
 #include <Common/Config/ConfigProcessor.h>
-#include <daemon/OwnSplitChannel.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/time.h>
@@ -23,18 +21,13 @@
 #include <sstream>
 #include <memory>
 #include <Poco/Observer.h>
-#include <Poco/Logger.h>
 #include <Poco/AutoPtr.h>
 #include <common/getThreadNumber.h>
 #include <Poco/PatternFormatter.h>
-#include <Poco/ConsoleChannel.h>
 #include <Poco/TaskManager.h>
 #include <Poco/File.h>
 #include <Poco/Path.h>
 #include <Poco/Message.h>
-#include <Poco/Util/AbstractConfiguration.h>
-#include <Poco/Util/XMLConfiguration.h>
-#include <Poco/Util/MapConfiguration.h>
 #include <Poco/Util/Application.h>
 #include <Poco/Exception.h>
 #include <Poco/ErrorHandler.h>
@@ -49,9 +42,7 @@
 #include <Common/getMultipleKeysFromConfig.h>
 #include <Common/ClickHouseRevision.h>
 #include <Common/config_version.h>
-#include <daemon/OwnPatternFormatter.h>
-#include <Common/CurrentThread.h>
-#include <Poco/Net/RemoteSyslogChannel.h>
+#include <common/argsToConfig.h>
 
 #if USE_UNWIND
     #define UNW_LOCAL_ONLY
@@ -257,7 +248,7 @@ public:
             else if (sig == SIGHUP || sig == SIGUSR1)
             {
                 LOG_DEBUG(log, "Received signal to close logs.");
-                BaseDaemon::instance().closeLogs();
+                BaseDaemon::instance().closeLogs(BaseDaemon::instance().logger());
                 LOG_INFO(log, "Opened new log file after received signal.");
             }
             else if (sig == Signals::StdTerminate)
@@ -572,6 +563,7 @@ static std::string createDirectory(const std::string & file)
     return path.toString();
 };
 
+
 static bool tryCreateDirectories(Poco::Logger * logger, const std::string & path)
 {
     try
@@ -767,146 +759,6 @@ void BaseDaemon::wakeup()
     wakeup_event.set();
 }
 
-
-void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config)
-{
-    auto current_logger = config.getString("logger");
-    if (config_logger == current_logger)
-        return;
-    config_logger = current_logger;
-
-    bool is_daemon = config.getBool("application.runAsDaemon", false);
-
-    /// Split logs to ordinary log, error log, syslog and console.
-    /// Use extended interface of Channel for more comprehensive logging.
-    Poco::AutoPtr<DB::OwnSplitChannel> split = new DB::OwnSplitChannel;
-
-    auto log_level = config.getString("logger.level", "trace");
-    const auto log_path = config.getString("logger.log", "");
-    if (!log_path.empty())
-    {
-        createDirectory(log_path);
-        std::cerr << "Logging " << log_level << " to " << log_path << std::endl;
-
-        // Set up two channel chains.
-        log_file = new Poco::FileChannel;
-        log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(log_path).absolute().toString());
-        log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M"));
-        log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number");
-        log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true"));
-        log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1"));
-        log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true"));
-        log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config.getRawString("logger.rotateOnOpen", "false"));
-        log_file->open();
-
-        Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this);
-
-        Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, log_file);
-        split->addChannel(log);
-    }
-
-    const auto errorlog_path = config.getString("logger.errorlog", "");
-    if (!errorlog_path.empty())
-    {
-        createDirectory(errorlog_path);
-        std::cerr << "Logging errors to " << errorlog_path << std::endl;
-
-        error_log_file = new Poco::FileChannel;
-        error_log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(errorlog_path).absolute().toString());
-        error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M"));
-        error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number");
-        error_log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true"));
-        error_log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1"));
-        error_log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true"));
-        error_log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config.getRawString("logger.rotateOnOpen", "false"));
-
-        Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this);
-
-        Poco::AutoPtr<DB::OwnFormattingChannel> errorlog = new DB::OwnFormattingChannel(pf, error_log_file);
-        errorlog->setLevel(Poco::Message::PRIO_NOTICE);
-        errorlog->open();
-        split->addChannel(errorlog);
-    }
-
-    /// "dynamic_layer_selection" is needed only for Yandex.Metrika, that share part of ClickHouse code.
-    /// We don't need this configuration parameter.
-
-    if (config.getBool("logger.use_syslog", false) || config.getBool("dynamic_layer_selection", false))
-    {
-        const std::string & cmd_name = commandName();
-
-        if (config.has("logger.syslog.address"))
-        {
-            syslog_channel = new Poco::Net::RemoteSyslogChannel();
-            // syslog address
-            syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_LOGHOST, config.getString("logger.syslog.address"));
-            if (config.has("logger.syslog.hostname"))
-            {
-                syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_HOST, config.getString("logger.syslog.hostname"));
-            }
-            syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_FORMAT, config.getString("logger.syslog.format", "syslog"));
-            syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_FACILITY, config.getString("logger.syslog.facility", "LOG_USER"));
-        }
-        else
-        {
-            syslog_channel = new Poco::SyslogChannel();
-            syslog_channel->setProperty(Poco::SyslogChannel::PROP_NAME, cmd_name);
-            syslog_channel->setProperty(Poco::SyslogChannel::PROP_OPTIONS, config.getString("logger.syslog.options", "LOG_CONS|LOG_PID"));
-            syslog_channel->setProperty(Poco::SyslogChannel::PROP_FACILITY, config.getString("logger.syslog.facility", "LOG_DAEMON"));
-        }
-        syslog_channel->open();
-
-        Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this, OwnPatternFormatter::ADD_LAYER_TAG);
-
-        Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, syslog_channel);
-        split->addChannel(log);
-    }
-
-    if (config.getBool("logger.console", false) || (!config.hasProperty("logger.console") && !is_daemon && (isatty(STDIN_FILENO) || isatty(STDERR_FILENO))))
-    {
-        Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(new OwnPatternFormatter(this), new Poco::ConsoleChannel);
-        logger().warning("Logging " + log_level + " to console");
-        split->addChannel(log);
-    }
-
-    split->open();
-    logger().close();
-    logger().setChannel(split);
-
-    // Global logging level (it can be overridden for specific loggers).
-    logger().setLevel(log_level);
-
-    // Set level to all already created loggers
-    std::vector <std::string> names;
-    Logger::root().names(names);
-    for (const auto & name : names)
-        Logger::root().get(name).setLevel(log_level);
-
-    // Attach to the root logger.
-    Logger::root().setLevel(log_level);
-    Logger::root().setChannel(logger().getChannel());
-
-    // Explicitly specified log levels for specific loggers.
-    Poco::Util::AbstractConfiguration::Keys levels;
-    config.keys("logger.levels", levels);
-
-    if (!levels.empty())
-        for (const auto & level : levels)
-            Logger::get(level).setLevel(config.getString("logger.levels." + level, "trace"));
-}
-
-
-void BaseDaemon::closeLogs()
-{
-    if (log_file)
-        log_file->close();
-    if (error_log_file)
-        error_log_file->close();
-
-    if (!log_file)
-        logger().warning("Logging to console but received signal to close log file (ignoring).");
-}
-
 std::string BaseDaemon::getDefaultCorePath() const
 {
     return "/opt/cores/";
@@ -951,62 +803,8 @@ void BaseDaemon::initialize(Application & self)
     task_manager.reset(new Poco::TaskManager);
     ServerApplication::initialize(self);
 
-    {
-        /// Parsing all args and converting to config layer
-        /// Test: -- --1=1 --1=2 --3 5 7 8 -9 10 -11=12 14= 15== --16==17 --=18 --19= --20 21 22 --23 --24 25 --26 -27 28 ---29=30 -- ----31 32 --33 3-4
-        Poco::AutoPtr<Poco::Util::MapConfiguration> map_config = new Poco::Util::MapConfiguration;
-        std::string key;
-        for(auto & arg : argv())
-        {
-            auto key_start = arg.find_first_not_of('-');
-            auto pos_minus = arg.find('-');
-            auto pos_eq = arg.find('=');
-
-            // old saved '--key', will set to some true value "1"
-            if (!key.empty() && pos_minus != std::string::npos && pos_minus < key_start)
-            {
-                map_config->setString(key, "1");
-                key = "";
-            }
-
-            if (pos_eq == std::string::npos)
-            {
-                if (!key.empty())
-                {
-                    if (pos_minus == std::string::npos || pos_minus > key_start)
-                    {
-                        map_config->setString(key, arg);
-                    }
-                    key = "";
-                }
-                if (pos_minus != std::string::npos && key_start != std::string::npos && pos_minus < key_start)
-                    key = arg.substr(key_start);
-                continue;
-            }
-            else
-            {
-                key = "";
-            }
-
-            if (key_start == std::string::npos)
-                continue;
-
-            if (pos_minus > key_start)
-                continue;
-
-            key = arg.substr(key_start, pos_eq - key_start);
-            if (key.empty())
-                continue;
-            std::string value;
-            if (arg.size() > pos_eq)
-                value = arg.substr(pos_eq+1);
-
-            map_config->setString(key, value);
-            key = "";
-        }
-        /// now highest priority (lowest value) is PRIO_APPLICATION = -100, we want higher!
-        config().add(map_config, PRIO_APPLICATION - 100);
-    }
+    /// now highest priority (lowest value) is PRIO_APPLICATION = -100, we want higher!
+    argsToConfig(argv(), config(), PRIO_APPLICATION - 100);
 
     bool is_daemon = config().getBool("application.runAsDaemon", false);
 
@@ -1101,7 +899,7 @@ void BaseDaemon::initialize(Application & self)
             throw Poco::Exception("Cannot change directory to /tmp");
     }
 
-    buildLoggers(config());
+    buildLoggers(config(), logger());
 
     if (is_daemon)
     {

libs/libloggers/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory(loggers)

libs/libloggers/loggers/CMakeLists.txt

@@ -0,0 +1,5 @@
+include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
+add_headers_and_sources(loggers .)
+add_library(loggers ${loggers_sources} ${loggers_headers})
+target_link_libraries(loggers PRIVATE dbms clickhouse_common_io ${Poco_Foundation_LIBRARY})
+target_include_directories(loggers PUBLIC ..)

libs/libloggers/loggers/ExtendedLogChannel.cpp

@@ -1,13 +1,13 @@
-#include <daemon/ExtendedLogChannel.h>
-#include <Common/Exception.h>
-#include <Common/CurrentThread.h>
-#include <common/getThreadNumber.h>
+#include "ExtendedLogChannel.h"
+
 #include <sys/time.h>
+#include <Common/CurrentThread.h>
+#include <Common/Exception.h>
+#include <common/getThreadNumber.h>
 
 
 namespace DB
 {
-
 namespace ErrorCodes
 {
     extern const int CANNOT_GETTIMEOFDAY;

libs/libloggers/loggers/ExtendedLogChannel.h

@@ -1,11 +1,13 @@
 #pragma once
-#include <Core/Types.h>
-#include <Poco/Message.h>
+#include <string>
 
+namespace Poco
+{
+class Message;
+}
 
 namespace DB
 {
-
 /// Poco::Message with more ClickHouse-specific info
 /// NOTE: Poco::Message is not polymorphic class, so we can't use inheritance in couple with dynamic_cast<>()
 class ExtendedLogMessage
@@ -19,10 +21,10 @@ public:
     // Do not copy for efficiency reasons
     const Poco::Message & base;
 
-    UInt32 time_seconds = 0;
-    UInt32 time_microseconds = 0;
+    uint32_t time_seconds = 0;
+    uint32_t time_microseconds = 0;
 
-    UInt32 thread_number = 0;
+    uint32_t thread_number = 0;
     std::string query_id;
 };
 

libs/libloggers/loggers/Loggers.cpp

@@ -0,0 +1,164 @@
+#include "Loggers.h"
+
+#include <iostream>
+#include <Poco/SyslogChannel.h>
+#include <Poco/Util/AbstractConfiguration.h>
+#include "OwnFormattingChannel.h"
+#include "OwnPatternFormatter.h"
+#include "OwnSplitChannel.h"
+#include <Poco/ConsoleChannel.h>
+#include <Poco/File.h>
+#include <Poco/Logger.h>
+#include <Poco/Net/RemoteSyslogChannel.h>
+#include <Poco/Path.h>
+
+// TODO: move to libcommon
+static std::string createDirectory(const std::string & file)
+{
+    auto path = Poco::Path(file).makeParent();
+    if (path.toString().empty())
+        return "";
+    Poco::File(path).createDirectories();
+    return path.toString();
+};
+
+void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger /*_root*/, const std::string & cmd_name)
+{
+    auto current_logger = config.getString("logger", "");
+    if (config_logger == current_logger)
+        return;
+    config_logger = current_logger;
+
+    bool is_daemon = config.getBool("application.runAsDaemon", false);
+
+    /// Split logs to ordinary log, error log, syslog and console.
+    /// Use extended interface of Channel for more comprehensive logging.
+    Poco::AutoPtr<DB::OwnSplitChannel> split = new DB::OwnSplitChannel;
+
+    auto log_level = config.getString("logger.level", "trace");
+    const auto log_path = config.getString("logger.log", "");
+    if (!log_path.empty())
+    {
+        createDirectory(log_path);
+        std::cerr << "Logging " << log_level << " to " << log_path << std::endl;
+
+        // Set up two channel chains.
+        log_file = new Poco::FileChannel;
+        log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(log_path).absolute().toString());
+        log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M"));
+        log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number");
+        log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true"));
+        log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1"));
+        log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true"));
+        log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config.getRawString("logger.rotateOnOpen", "false"));
+        log_file->open();
+
+        Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this);
+
+        Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, log_file);
+        split->addChannel(log);
+    }
+
+    const auto errorlog_path = config.getString("logger.errorlog", "");
+    if (!errorlog_path.empty())
+    {
+        createDirectory(errorlog_path);
+        std::cerr << "Logging errors to " << errorlog_path << std::endl;
+
+        error_log_file = new Poco::FileChannel;
+        error_log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(errorlog_path).absolute().toString());
+        error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M"));
+        error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number");
+        error_log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true"));
+        error_log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1"));
+        error_log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true"));
+        error_log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config.getRawString("logger.rotateOnOpen", "false"));
+
+        Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this);
+
+        Poco::AutoPtr<DB::OwnFormattingChannel> errorlog = new DB::OwnFormattingChannel(pf, error_log_file);
+        errorlog->setLevel(Poco::Message::PRIO_NOTICE);
+        errorlog->open();
+        split->addChannel(errorlog);
+    }
+
+    /// "dynamic_layer_selection" is needed only for Yandex.Metrika, that share part of ClickHouse code.
+    /// We don't need this configuration parameter.
+
+    if (config.getBool("logger.use_syslog", false) || config.getBool("dynamic_layer_selection", false))
+    {
+        //const std::string & cmd_name = commandName();
+
+        if (config.has("logger.syslog.address"))
+        {
+            syslog_channel = new Poco::Net::RemoteSyslogChannel();
+            // syslog address
+            syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_LOGHOST, config.getString("logger.syslog.address"));
+            if (config.has("logger.syslog.hostname"))
+            {
+                syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_HOST, config.getString("logger.syslog.hostname"));
+            }
+            syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_FORMAT, config.getString("logger.syslog.format", "syslog"));
+            syslog_channel->setProperty(
+                Poco::Net::RemoteSyslogChannel::PROP_FACILITY, config.getString("logger.syslog.facility", "LOG_USER"));
+        }
+        else
+        {
+            syslog_channel = new Poco::SyslogChannel();
+            syslog_channel->setProperty(Poco::SyslogChannel::PROP_NAME, cmd_name);
+            syslog_channel->setProperty(Poco::SyslogChannel::PROP_OPTIONS, config.getString("logger.syslog.options", "LOG_CONS|LOG_PID"));
+            syslog_channel->setProperty(Poco::SyslogChannel::PROP_FACILITY, config.getString("logger.syslog.facility", "LOG_DAEMON"));
+        }
+        syslog_channel->open();
+
+        Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this, OwnPatternFormatter::ADD_LAYER_TAG);
+
+        Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, syslog_channel);
+        split->addChannel(log);
+    }
+
+    if (config.getBool("logger.console", false)
+        || (!config.hasProperty("logger.console") && !is_daemon && (isatty(STDIN_FILENO) || isatty(STDERR_FILENO))))
+    {
+        Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(new OwnPatternFormatter(this), new Poco::ConsoleChannel);
+        logger.warning("Logging " + log_level + " to console");
+        split->addChannel(log);
+    }
+
+    split->open();
+    logger.close();
+    logger.setChannel(split);
+
+    // Global logging level (it can be overridden for specific loggers).
+    logger.setLevel(log_level);
+
+    // Set level to all already created loggers
+    std::vector<std::string> names;
+    //logger_root = Logger::root();
+    logger.root().names(names);
+    for (const auto & name : names)
+        logger.root().get(name).setLevel(log_level);
+
+    // Attach to the root logger.
+    logger.root().setLevel(log_level);
+    logger.root().setChannel(logger.getChannel());
+
+    // Explicitly specified log levels for specific loggers.
+    Poco::Util::AbstractConfiguration::Keys levels;
+    config.keys("logger.levels", levels);
+
+    if (!levels.empty())
+        for (const auto & level : levels)
+            logger.root().get(level).setLevel(config.getString("logger.levels." + level, "trace"));
+}
+
+void Loggers::closeLogs(Poco::Logger & logger)
+{
+    if (log_file)
+        log_file->close();
+    if (error_log_file)
+        error_log_file->close();
+
+    if (!log_file)
+        logger.warning("Logging to console but received signal to close log file (ignoring).");
+}

libs/libloggers/loggers/Loggers.h

@@ -0,0 +1,37 @@
+#include <optional>
+#include <string>
+#include <Poco/AutoPtr.h>
+#include <Poco/FileChannel.h>
+#include <Poco/Util/Application.h>
+
+namespace Poco::Util
+{
+class AbstractConfiguration;
+}
+
+
+class Loggers
+{
+public:
+    /// Строит необходимые логгеры
+    void buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger, const std::string & cmd_name = "");
+
+    /// Закрыть файлы с логами. При следующей записи, будут созданы новые файлы.
+    void closeLogs(Poco::Logger & logger);
+
+    std::optional<size_t> getLayer() const
+    {
+        return layer; /// layer выставляется в классе-наследнике BaseDaemonApplication.
+    }
+
+protected:
+    std::optional<size_t> layer;
+
+private:
+    /// Файлы с логами.
+    Poco::AutoPtr<Poco::FileChannel> log_file;
+    Poco::AutoPtr<Poco::FileChannel> error_log_file;
+    Poco::AutoPtr<Poco::Channel> syslog_channel;
+    /// Previous value of logger element in config. It is used to reinitialize loggers whenever the value changed.
+    std::string config_logger;
+};

libs/libloggers/loggers/OwnFormattingChannel.cpp

@@ -1,10 +1,9 @@
-#include <daemon/OwnFormattingChannel.h>
-#include <daemon/OwnPatternFormatter.h>
+#include "OwnFormattingChannel.h"
+#include "OwnPatternFormatter.h"
 
 
 namespace DB
 {
-
 void OwnFormattingChannel::logExtended(const ExtendedLogMessage & msg)
 {
     if (pChannel && priority >= msg.base.getPriority())

libs/libloggers/loggers/OwnFormattingChannel.h

@@ -1,31 +1,26 @@
 #pragma once
-#include <daemon/ExtendedLogChannel.h>
 #include <Poco/AutoPtr.h>
 #include <Poco/Channel.h>
 #include <Poco/FormattingChannel.h>
+#include "ExtendedLogChannel.h"
 #include "OwnPatternFormatter.h"
 
 
 namespace DB
 {
-
-
 // Like Poco::FormattingChannel but supports the extended logging interface and log level filter
 class OwnFormattingChannel : public Poco::Channel, public ExtendedLogChannel
 {
 public:
-    explicit OwnFormattingChannel(Poco::AutoPtr<OwnPatternFormatter> pFormatter_ = nullptr, Poco::AutoPtr<Poco::Channel> pChannel_ = nullptr)
-        : pFormatter(std::move(pFormatter_)), pChannel(std::move(pChannel_)) {}
-
-    void setChannel(Poco::AutoPtr<Poco::Channel> pChannel_)
+    explicit OwnFormattingChannel(
+        Poco::AutoPtr<OwnPatternFormatter> pFormatter_ = nullptr, Poco::AutoPtr<Poco::Channel> pChannel_ = nullptr)
+        : pFormatter(std::move(pFormatter_)), pChannel(std::move(pChannel_))
     {
-        pChannel = std::move(pChannel_);
     }
 
-    void setLevel(Poco::Message::Priority priority_)
-    {
-        priority = priority_;
-    }
+    void setChannel(Poco::AutoPtr<Poco::Channel> pChannel_) { pChannel = std::move(pChannel_); }
+
+    void setLevel(Poco::Message::Priority priority_) { priority = priority_; }
 
     void open() override
     {

libs/libloggers/loggers/OwnPatternFormatter.cpp

@@ -1,19 +1,20 @@
-#include <daemon/OwnPatternFormatter.h>
-
-#include <IO/WriteBufferFromString.h>
-#include <IO/WriteHelpers.h>
-#include <Common/CurrentThread.h>
-#include <Interpreters/InternalTextLogsQueue.h>
+#include "OwnPatternFormatter.h"
 
 #include <functional>
 #include <optional>
+#include <IO/WriteBufferFromString.h>
+#include <IO/WriteHelpers.h>
+#include <Interpreters/InternalTextLogsQueue.h>
 #include <sys/time.h>
+#include <Common/CurrentThread.h>
 #include <common/getThreadNumber.h>
-#include <daemon/BaseDaemon.h>
+#include "Loggers.h"
 
 
-OwnPatternFormatter::OwnPatternFormatter(const BaseDaemon * daemon_, OwnPatternFormatter::Options options_)
-    : Poco::PatternFormatter(""), daemon(daemon_), options(options_) {}
+OwnPatternFormatter::OwnPatternFormatter(const /*BaseDaemon*/ Loggers * daemon_, OwnPatternFormatter::Options options_)
+    : Poco::PatternFormatter(""), daemon(daemon_), options(options_)
+{
+}
 
 
 void OwnPatternFormatter::formatExtended(const DB::ExtendedLogMessage & msg_ext, std::string & text)

libs/libloggers/loggers/OwnPatternFormatter.h

@@ -2,7 +2,7 @@
 
 
 #include <Poco/PatternFormatter.h>
-#include <daemon/ExtendedLogChannel.h>
+#include "ExtendedLogChannel.h"
 
 
 /** Форматирует по своему.
@@ -19,12 +19,11 @@
   * Также сделан чуть более эффективным (что имеет мало значения).
   */
 
-class BaseDaemon;
+class Loggers;
 
 class OwnPatternFormatter : public Poco::PatternFormatter
 {
 public:
-
     /// ADD_LAYER_TAG is needed only for Yandex.Metrika, that share part of ClickHouse code.
     enum Options
     {
@@ -32,12 +31,12 @@ public:
         ADD_LAYER_TAG = 1 << 0
     };
 
-    OwnPatternFormatter(const BaseDaemon * daemon_, Options options_ = ADD_NOTHING);
+    OwnPatternFormatter(const Loggers * daemon_, Options options_ = ADD_NOTHING);
 
     void format(const Poco::Message & msg, std::string & text) override;
     void formatExtended(const DB::ExtendedLogMessage & msg_ext, std::string & text);
 
 private:
-    const BaseDaemon * daemon;
+    const Loggers * daemon;
     Options options;
 };

libs/libloggers/loggers/OwnSplitChannel.cpp

@@ -1,21 +1,17 @@
-#include <daemon/OwnSplitChannel.h>
+#include "OwnSplitChannel.h"
 
+#include <iostream>
+#include <Core/Block.h>
+#include <Interpreters/InternalTextLogsQueue.h>
+#include <sys/time.h>
+#include <Poco/Message.h>
 #include <Common/CurrentThread.h>
 #include <Common/DNSResolver.h>
-#include <Interpreters/InternalTextLogsQueue.h>
-#include <Core/Block.h>
-
-#include <Poco/Message.h>
 #include <common/getThreadNumber.h>
 
-#include <sys/time.h>
-#include <iostream>
-
 
 namespace DB
 {
-
-
 void OwnSplitChannel::log(const Poco::Message & msg)
 {
     auto logs_queue = CurrentThread::getInternalTextLogsQueue();

libs/libloggers/loggers/OwnSplitChannel.h

@@ -1,13 +1,12 @@
 #pragma once
-#include <daemon/ExtendedLogChannel.h>
-#include <Poco/Channel.h>
-#include <Poco/AutoPtr.h>
 #include <vector>
+#include <Poco/AutoPtr.h>
+#include <Poco/Channel.h>
+#include "ExtendedLogChannel.h"
 
 
 namespace DB
 {
-
 /// Works as Poco::SplitterChannel, but performs additional work:
 ///  passes logs to Client via TCP interface
 ///  tries to use extended logging interface of child for more comprehensive logging

utils/build/build_macos.sh

@@ -14,6 +14,10 @@ fi
 
 brew install cmake ninja gcc icu4c mariadb-connector-c openssl unixodbc libtool gettext readline librdkafka
 
+# If you want to run tests
+brew install python
+sudo pip install lxml termcolor requests
+
 ## Checkout ClickHouse sources
 
 #  To get the latest stable version: