diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 79e6a18e00e..2c421cc20b3 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -65,6 +65,7 @@ #include #include #include +#include #if USE_READLINE #include "Suggest.h" @@ -1549,7 +1550,7 @@ public: * where possible args are file, name, format, structure, types. * Split these groups before processing. */ - using Arguments = std::vector; + using Arguments = std::vector; Arguments common_arguments{""}; /// 0th argument is ignored. std::vector external_tables_arguments; @@ -1671,8 +1672,7 @@ public: ("types", po::value(), "types") ; /// Parse main commandline options. - po::parsed_options parsed = po::command_line_parser( - common_arguments.size(), common_arguments.data()).options(main_description).run(); + po::parsed_options parsed = po::command_line_parser(common_arguments).options(main_description).run(); po::variables_map options; po::store(parsed, options); po::notify(options); @@ -1705,8 +1705,7 @@ public: for (size_t i = 0; i < external_tables_arguments.size(); ++i) { /// Parse commandline options related to external tables. - po::parsed_options parsed_tables = po::command_line_parser( - external_tables_arguments[i].size(), external_tables_arguments[i].data()).options(external_description).run(); + po::parsed_options parsed_tables = po::command_line_parser(external_tables_arguments[i]).options(external_description).run(); po::variables_map external_options; po::store(parsed_tables, external_options); @@ -1802,6 +1801,9 @@ public: } if (options.count("suggestion_limit")) config().setInt("suggestion_limit", options["suggestion_limit"].as()); + + argsToConfig(common_arguments, config(), 100); + } }; diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp index 5d388686d55..9cd3629192f 100644 --- a/dbms/programs/copier/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include diff --git a/dbms/programs/local/CMakeLists.txt b/dbms/programs/local/CMakeLists.txt index 299458ef913..41780936977 100644 --- a/dbms/programs/local/CMakeLists.txt +++ b/dbms/programs/local/CMakeLists.txt @@ -6,4 +6,4 @@ clickhouse_program_add(local) if(NOT CLICKHOUSE_ONE_SHARED) target_link_libraries(clickhouse-local-lib PRIVATE clickhouse-server-lib) -endif () +endif() diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index 73e43dffec3..330404aa2e6 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -59,11 +59,29 @@ void LocalServer::initialize(Poco::Util::Application & self) { Poco::Util::Application::initialize(self); - // Turn off server logging to stderr - if (!config().has("verbose")) + /// Load config files if exists + if (config().has("config-file") || Poco::File("config.xml").exists()) { - Poco::Logger::root().setLevel("none"); - Poco::Logger::root().setChannel(Poco::AutoPtr(new Poco::NullChannel())); + const auto config_path = config().getString("config-file", "config.xml"); + ConfigProcessor config_processor(config_path, false, true); + config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString()); + auto loaded_config = config_processor.loadConfig(); + config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", ".")); + config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); + } + + if (config().has("logger") || config().has("logger.level") || config().has("logger.log")) + { + buildLoggers(config(), logger()); + } + else + { + // Turn off server logging to stderr + if (!config().has("verbose")) + { + Poco::Logger::root().setLevel("none"); + Poco::Logger::root().setChannel(Poco::AutoPtr(new Poco::NullChannel())); + } } } @@ -110,16 +128,6 @@ try return Application::EXIT_OK; } - /// Load config files if exists - if (config().has("config-file") || Poco::File("config.xml").exists()) - { - const auto config_path = config().getString("config-file", "config.xml"); - ConfigProcessor config_processor(config_path, false, true); - config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString()); - auto loaded_config = config_processor.loadConfig(); - config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", DBMS_DEFAULT_PATH)); - config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); - } context = std::make_unique(Context::createGlobal()); context->setGlobalContext(*context); @@ -428,6 +436,8 @@ void LocalServer::init(int argc, char ** argv) ("stacktrace", "print stack traces of exceptions") ("echo", "print query before execution") ("verbose", "print query and other debugging info") + ("logger.log", po::value(), "Log file name") + ("logger.level", po::value(), "Log level") ("ignore-error", "do not stop processing if a query failed") ("version,V", "print version information and exit") ; @@ -481,6 +491,10 @@ void LocalServer::init(int argc, char ** argv) config().setBool("echo", true); if (options.count("verbose")) config().setBool("verbose", true); + if (options.count("logger.log")) + config().setString("logger.log", options["logger.log"].as()); + if (options.count("logger.level")) + config().setString("logger.level", options["logger.level"].as()); if (options.count("ignore-error")) config().setBool("ignore-error", true); } diff --git a/dbms/programs/local/LocalServer.h b/dbms/programs/local/LocalServer.h index 2f59778490c..a79ab484107 100644 --- a/dbms/programs/local/LocalServer.h +++ b/dbms/programs/local/LocalServer.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -13,7 +14,7 @@ class Context; /// Lightweight Application for clickhouse-local /// No networking, no extra configs and working directories, no pid and status files, no dictionaries, no logging. /// Quiet mode by default -class LocalServer : public Poco::Util::Application +class LocalServer : public Poco::Util::Application, public Loggers { public: LocalServer(); diff --git a/dbms/programs/odbc-bridge/ODBCBridge.cpp b/dbms/programs/odbc-bridge/ODBCBridge.cpp index a6324b6e229..aaacdfca826 100644 --- a/dbms/programs/odbc-bridge/ODBCBridge.cpp +++ b/dbms/programs/odbc-bridge/ODBCBridge.cpp @@ -123,7 +123,7 @@ void ODBCBridge::initialize(Application & self) config().setString("logger", "ODBCBridge"); - buildLoggers(config()); + buildLoggers(config(), logger()); log = &logger(); hostname = config().getString("listen-host", "localhost"); port = config().getUInt("http-port"); diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 68520112565..91203e4ee7c 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -405,7 +405,7 @@ int Server::main(const std::vector & /*args*/) main_config_zk_changed_event, [&](ConfigurationPtr config) { - buildLoggers(*config); + buildLoggers(*config, logger()); global_context->setClustersConfig(config); global_context->setMacros(std::make_unique(*config, "macros")); }, diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index fcef7e6a8d1..6ec9ec5d416 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -1,8 +1,6 @@ #include #include #include -#include - #include #include #include diff --git a/dbms/tests/queries/shell_config.sh b/dbms/tests/queries/shell_config.sh index 7126a99b348..d4ab11be927 100644 --- a/dbms/tests/queries/shell_config.sh +++ b/dbms/tests/queries/shell_config.sh @@ -22,7 +22,7 @@ export CLICKHOUSE_CONFIG_CLIENT=${CLICKHOUSE_CONFIG_CLIENT:="/etc/clickhouse-cli [ -x "${CLICKHOUSE_BINARY}" ] && CLICKHOUSE_EXTRACT_CONFIG=${CLICKHOUSE_EXTRACT_CONFIG:="$CLICKHOUSE_BINARY extract-from-config --config=$CLICKHOUSE_CONFIG"} export CLICKHOUSE_EXTRACT_CONFIG=${CLICKHOUSE_EXTRACT_CONFIG:="$CLICKHOUSE_BINARY-extract-from-config --config=$CLICKHOUSE_CONFIG"} -[ -x "${CLICKHOUSE_BINARY}-format" ] && CLICKHOUSE_FORMAT=${CLICKHOUSE_FORMAT=:="$CLICKHOUSE_BINARY-format"} +[ -x "${CLICKHOUSE_BINARY}-format" ] && CLICKHOUSE_FORMAT=${CLICKHOUSE_FORMAT:="$CLICKHOUSE_BINARY-format"} [ -x "${CLICKHOUSE_BINARY}" ] && CLICKHOUSE_FORMAT=${CLICKHOUSE_FORMAT:="$CLICKHOUSE_BINARY format"} export CLICKHOUSE_FORMAT=${CLICKHOUSE_FORMAT:="$CLICKHOUSE_BINARY-format"} diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index 89c8748df4c..d526a662dc0 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -8,6 +8,7 @@ if (USE_DEBUG_HELPERS) endif () add_subdirectory (libcommon) +add_subdirectory (libloggers) add_subdirectory (libdaemon) if (USE_INTERNAL_MEMCPY) diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index 7c124b77230..6644e2f527e 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -21,6 +21,7 @@ add_library(common src/demangle.cpp src/setTerminalEcho.cpp src/getThreadNumber.cpp + src/argsToConfig.cpp include/common/Types.h include/common/DayNum.h @@ -107,6 +108,7 @@ endif() target_link_libraries (common PUBLIC + ${Poco_Util_LIBRARY} ${Poco_Foundation_LIBRARY} ${CITYHASH_LIBRARIES} PRIVATE diff --git a/libs/libcommon/include/common/argsToConfig.h b/libs/libcommon/include/common/argsToConfig.h new file mode 100644 index 00000000000..1c1607bc4c5 --- /dev/null +++ b/libs/libcommon/include/common/argsToConfig.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace Poco::Util +{ +class LayeredConfiguration; +} + +/// Import extra command line arguments to configuration. These are command line arguments after --. +void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::LayeredConfiguration & config, int priority); diff --git a/libs/libcommon/src/argsToConfig.cpp b/libs/libcommon/src/argsToConfig.cpp new file mode 100644 index 00000000000..b0ec2900268 --- /dev/null +++ b/libs/libcommon/src/argsToConfig.cpp @@ -0,0 +1,67 @@ +#include + +#include +#include +#include + + +void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::LayeredConfiguration & config, int priority) +{ + /// Parsing all args and converting to config layer + /// Test: -- --1=1 --1=2 --3 5 7 8 -9 10 -11=12 14= 15== --16==17 --=18 --19= --20 21 22 --23 --24 25 --26 -27 28 ---29=30 -- ----31 32 --33 3-4 + Poco::AutoPtr map_config = new Poco::Util::MapConfiguration; + std::string key; + for (auto & arg : argv) + { + auto key_start = arg.find_first_not_of('-'); + auto pos_minus = arg.find('-'); + auto pos_eq = arg.find('='); + + // old saved '--key', will set to some true value "1" + if (!key.empty() && pos_minus != std::string::npos && pos_minus < key_start) + { + map_config->setString(key, "1"); + key = ""; + } + + if (pos_eq == std::string::npos) + { + if (!key.empty()) + { + if (pos_minus == std::string::npos || pos_minus > key_start) + { + map_config->setString(key, arg); + } + key = ""; + } + if (pos_minus != std::string::npos && key_start != std::string::npos && pos_minus < key_start) + key = arg.substr(key_start); + continue; + } + else + { + key = ""; + } + + if (key_start == std::string::npos) + continue; + + if (pos_minus > key_start) + continue; + + key = arg.substr(key_start, pos_eq - key_start); + if (key.empty()) + continue; + std::string value; + if (arg.size() > pos_eq) + value = arg.substr(pos_eq + 1); + + map_config->setString(key, value); + key = ""; + } + + Poco::Util::MapConfiguration::Keys keys; + map_config->keys(keys); + + config.add(map_config, priority); +} diff --git a/libs/libdaemon/CMakeLists.txt b/libs/libdaemon/CMakeLists.txt index a7839dc158f..d74a78fff31 100644 --- a/libs/libdaemon/CMakeLists.txt +++ b/libs/libdaemon/CMakeLists.txt @@ -1,17 +1,9 @@ add_library (daemon src/BaseDaemon.cpp src/GraphiteWriter.cpp - src/ExtendedLogChannel.cpp - src/OwnPatternFormatter.cpp - src/OwnFormattingChannel.cpp - src/OwnSplitChannel.cpp include/daemon/BaseDaemon.h include/daemon/GraphiteWriter.h - include/daemon/ExtendedLogChannel.h - include/daemon/OwnPatternFormatter.h - include/daemon/OwnFormattingChannel.h - include/daemon/OwnSplitChannel.h ) if (USE_UNWIND) @@ -22,4 +14,4 @@ endif () target_include_directories (daemon PUBLIC include) -target_link_libraries (daemon PRIVATE clickhouse_common_io clickhouse_common_config common ${Poco_Net_LIBRARY} ${Poco_Util_LIBRARY} ${EXECINFO_LIBRARIES}) +target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickhouse_common_config common ${Poco_Net_LIBRARY} ${Poco_Util_LIBRARY} ${EXECINFO_LIBRARIES}) diff --git a/libs/libdaemon/include/daemon/BaseDaemon.h b/libs/libdaemon/include/daemon/BaseDaemon.h index f9497e582a3..9d323492c1f 100644 --- a/libs/libdaemon/include/daemon/BaseDaemon.h +++ b/libs/libdaemon/include/daemon/BaseDaemon.h @@ -16,13 +16,12 @@ #include #include #include -#include -#include #include #include #include #include #include +#include namespace Poco { class TaskManager; } @@ -40,7 +39,7 @@ namespace Poco { class TaskManager; } /// /// You can configure different log options for different loggers used inside program /// by providing subsections to "logger" in configuration file. -class BaseDaemon : public Poco::Util::ServerApplication +class BaseDaemon : public Poco::Util::ServerApplication, public Loggers { friend class SignalListener; @@ -56,9 +55,6 @@ public: /// Читает конфигурацию void reloadConfiguration(); - /// Строит необходимые логгеры - void buildLoggers(Poco::Util::AbstractConfiguration & config); - /// Определяет параметр командной строки void defineOptions(Poco::Util::OptionSet & _options) override; @@ -92,9 +88,6 @@ public: /// Разбудить void wakeup(); - /// Закрыть файлы с логами. При следующей записи, будут созданы новые файлы. - void closeLogs(); - /// В Graphite компоненты пути(папки) разделяются точкой. /// У нас принят путь формата root_path.hostname_yandex_ru.key /// root_path по умолчанию one_min @@ -130,11 +123,6 @@ public: return nullptr; } - std::optional getLayer() const - { - return layer; /// layer выставляется в классе-наследнике BaseDaemonApplication. - } - /// close all process FDs except /// 0-2 -- stdin, stdout, stderr /// also doesn't close global internal pipes for signal handling @@ -209,15 +197,8 @@ protected: Poco::Thread signal_listener_thread; std::unique_ptr signal_listener; - /// Файлы с логами. - Poco::AutoPtr log_file; - Poco::AutoPtr error_log_file; - Poco::AutoPtr syslog_channel; - std::map> graphite_writers; - std::optional layer; - std::mutex signal_handler_mutex; std::condition_variable signal_event; std::atomic_size_t terminate_signals_counter{0}; @@ -229,9 +210,6 @@ protected: private: - /// Previous value of logger element in config. It is used to reinitialize loggers whenever the value changed. - std::string config_logger; - /// Check SSE and others instructions availability /// Calls exit on fail void checkRequiredInstructions(); diff --git a/libs/libdaemon/src/BaseDaemon.cpp b/libs/libdaemon/src/BaseDaemon.cpp index 11e7af40f51..596b738c0bb 100644 --- a/libs/libdaemon/src/BaseDaemon.cpp +++ b/libs/libdaemon/src/BaseDaemon.cpp @@ -1,8 +1,6 @@ #include -#include -#include + #include -#include #include #include #include @@ -23,18 +21,13 @@ #include #include #include -#include #include #include #include -#include #include #include #include #include -#include -#include -#include #include #include #include @@ -49,9 +42,7 @@ #include #include #include -#include -#include -#include +#include #if USE_UNWIND #define UNW_LOCAL_ONLY @@ -257,7 +248,7 @@ public: else if (sig == SIGHUP || sig == SIGUSR1) { LOG_DEBUG(log, "Received signal to close logs."); - BaseDaemon::instance().closeLogs(); + BaseDaemon::instance().closeLogs(BaseDaemon::instance().logger()); LOG_INFO(log, "Opened new log file after received signal."); } else if (sig == Signals::StdTerminate) @@ -572,6 +563,7 @@ static std::string createDirectory(const std::string & file) return path.toString(); }; + static bool tryCreateDirectories(Poco::Logger * logger, const std::string & path) { try @@ -767,146 +759,6 @@ void BaseDaemon::wakeup() wakeup_event.set(); } - -void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config) -{ - auto current_logger = config.getString("logger"); - if (config_logger == current_logger) - return; - config_logger = current_logger; - - bool is_daemon = config.getBool("application.runAsDaemon", false); - - /// Split logs to ordinary log, error log, syslog and console. - /// Use extended interface of Channel for more comprehensive logging. - Poco::AutoPtr split = new DB::OwnSplitChannel; - - auto log_level = config.getString("logger.level", "trace"); - const auto log_path = config.getString("logger.log", ""); - if (!log_path.empty()) - { - createDirectory(log_path); - std::cerr << "Logging " << log_level << " to " << log_path << std::endl; - - // Set up two channel chains. - log_file = new Poco::FileChannel; - log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(log_path).absolute().toString()); - log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M")); - log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); - log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true")); - log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1")); - log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true")); - log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config.getRawString("logger.rotateOnOpen", "false")); - log_file->open(); - - Poco::AutoPtr pf = new OwnPatternFormatter(this); - - Poco::AutoPtr log = new DB::OwnFormattingChannel(pf, log_file); - split->addChannel(log); - } - - const auto errorlog_path = config.getString("logger.errorlog", ""); - if (!errorlog_path.empty()) - { - createDirectory(errorlog_path); - std::cerr << "Logging errors to " << errorlog_path << std::endl; - - error_log_file = new Poco::FileChannel; - error_log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(errorlog_path).absolute().toString()); - error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M")); - error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); - error_log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true")); - error_log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1")); - error_log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true")); - error_log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config.getRawString("logger.rotateOnOpen", "false")); - - Poco::AutoPtr pf = new OwnPatternFormatter(this); - - Poco::AutoPtr errorlog = new DB::OwnFormattingChannel(pf, error_log_file); - errorlog->setLevel(Poco::Message::PRIO_NOTICE); - errorlog->open(); - split->addChannel(errorlog); - } - - /// "dynamic_layer_selection" is needed only for Yandex.Metrika, that share part of ClickHouse code. - /// We don't need this configuration parameter. - - if (config.getBool("logger.use_syslog", false) || config.getBool("dynamic_layer_selection", false)) - { - const std::string & cmd_name = commandName(); - - if (config.has("logger.syslog.address")) - { - syslog_channel = new Poco::Net::RemoteSyslogChannel(); - // syslog address - syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_LOGHOST, config.getString("logger.syslog.address")); - if (config.has("logger.syslog.hostname")) - { - syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_HOST, config.getString("logger.syslog.hostname")); - } - syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_FORMAT, config.getString("logger.syslog.format", "syslog")); - syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_FACILITY, config.getString("logger.syslog.facility", "LOG_USER")); - } - else - { - syslog_channel = new Poco::SyslogChannel(); - syslog_channel->setProperty(Poco::SyslogChannel::PROP_NAME, cmd_name); - syslog_channel->setProperty(Poco::SyslogChannel::PROP_OPTIONS, config.getString("logger.syslog.options", "LOG_CONS|LOG_PID")); - syslog_channel->setProperty(Poco::SyslogChannel::PROP_FACILITY, config.getString("logger.syslog.facility", "LOG_DAEMON")); - } - syslog_channel->open(); - - Poco::AutoPtr pf = new OwnPatternFormatter(this, OwnPatternFormatter::ADD_LAYER_TAG); - - Poco::AutoPtr log = new DB::OwnFormattingChannel(pf, syslog_channel); - split->addChannel(log); - } - - if (config.getBool("logger.console", false) || (!config.hasProperty("logger.console") && !is_daemon && (isatty(STDIN_FILENO) || isatty(STDERR_FILENO)))) - { - Poco::AutoPtr log = new DB::OwnFormattingChannel(new OwnPatternFormatter(this), new Poco::ConsoleChannel); - logger().warning("Logging " + log_level + " to console"); - split->addChannel(log); - } - - split->open(); - logger().close(); - logger().setChannel(split); - - // Global logging level (it can be overridden for specific loggers). - logger().setLevel(log_level); - - // Set level to all already created loggers - std::vector names; - Logger::root().names(names); - for (const auto & name : names) - Logger::root().get(name).setLevel(log_level); - - // Attach to the root logger. - Logger::root().setLevel(log_level); - Logger::root().setChannel(logger().getChannel()); - - // Explicitly specified log levels for specific loggers. - Poco::Util::AbstractConfiguration::Keys levels; - config.keys("logger.levels", levels); - - if (!levels.empty()) - for (const auto & level : levels) - Logger::get(level).setLevel(config.getString("logger.levels." + level, "trace")); -} - - -void BaseDaemon::closeLogs() -{ - if (log_file) - log_file->close(); - if (error_log_file) - error_log_file->close(); - - if (!log_file) - logger().warning("Logging to console but received signal to close log file (ignoring)."); -} - std::string BaseDaemon::getDefaultCorePath() const { return "/opt/cores/"; @@ -951,62 +803,8 @@ void BaseDaemon::initialize(Application & self) task_manager.reset(new Poco::TaskManager); ServerApplication::initialize(self); - { - /// Parsing all args and converting to config layer - /// Test: -- --1=1 --1=2 --3 5 7 8 -9 10 -11=12 14= 15== --16==17 --=18 --19= --20 21 22 --23 --24 25 --26 -27 28 ---29=30 -- ----31 32 --33 3-4 - Poco::AutoPtr map_config = new Poco::Util::MapConfiguration; - std::string key; - for(auto & arg : argv()) - { - auto key_start = arg.find_first_not_of('-'); - auto pos_minus = arg.find('-'); - auto pos_eq = arg.find('='); - - // old saved '--key', will set to some true value "1" - if (!key.empty() && pos_minus != std::string::npos && pos_minus < key_start) - { - map_config->setString(key, "1"); - key = ""; - } - - if (pos_eq == std::string::npos) - { - if (!key.empty()) - { - if (pos_minus == std::string::npos || pos_minus > key_start) - { - map_config->setString(key, arg); - } - key = ""; - } - if (pos_minus != std::string::npos && key_start != std::string::npos && pos_minus < key_start) - key = arg.substr(key_start); - continue; - } - else - { - key = ""; - } - - if (key_start == std::string::npos) - continue; - - if (pos_minus > key_start) - continue; - - key = arg.substr(key_start, pos_eq - key_start); - if (key.empty()) - continue; - std::string value; - if (arg.size() > pos_eq) - value = arg.substr(pos_eq+1); - - map_config->setString(key, value); - key = ""; - } - /// now highest priority (lowest value) is PRIO_APPLICATION = -100, we want higher! - config().add(map_config, PRIO_APPLICATION - 100); - } + /// now highest priority (lowest value) is PRIO_APPLICATION = -100, we want higher! + argsToConfig(argv(), config(), PRIO_APPLICATION - 100); bool is_daemon = config().getBool("application.runAsDaemon", false); @@ -1101,7 +899,7 @@ void BaseDaemon::initialize(Application & self) throw Poco::Exception("Cannot change directory to /tmp"); } - buildLoggers(config()); + buildLoggers(config(), logger()); if (is_daemon) { diff --git a/libs/libloggers/CMakeLists.txt b/libs/libloggers/CMakeLists.txt new file mode 100644 index 00000000000..3cc73da5bce --- /dev/null +++ b/libs/libloggers/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(loggers) diff --git a/libs/libloggers/loggers/CMakeLists.txt b/libs/libloggers/loggers/CMakeLists.txt new file mode 100644 index 00000000000..bada5a2fe1d --- /dev/null +++ b/libs/libloggers/loggers/CMakeLists.txt @@ -0,0 +1,5 @@ +include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) +add_headers_and_sources(loggers .) +add_library(loggers ${loggers_sources} ${loggers_headers}) +target_link_libraries(loggers PRIVATE dbms clickhouse_common_io ${Poco_Foundation_LIBRARY}) +target_include_directories(loggers PUBLIC ..) diff --git a/libs/libdaemon/src/ExtendedLogChannel.cpp b/libs/libloggers/loggers/ExtendedLogChannel.cpp similarity index 95% rename from libs/libdaemon/src/ExtendedLogChannel.cpp rename to libs/libloggers/loggers/ExtendedLogChannel.cpp index 7a1ac0688e0..857765a94de 100644 --- a/libs/libdaemon/src/ExtendedLogChannel.cpp +++ b/libs/libloggers/loggers/ExtendedLogChannel.cpp @@ -1,13 +1,13 @@ -#include -#include -#include -#include +#include "ExtendedLogChannel.h" + #include +#include +#include +#include namespace DB { - namespace ErrorCodes { extern const int CANNOT_GETTIMEOFDAY; diff --git a/libs/libdaemon/include/daemon/ExtendedLogChannel.h b/libs/libloggers/loggers/ExtendedLogChannel.h similarity index 82% rename from libs/libdaemon/include/daemon/ExtendedLogChannel.h rename to libs/libloggers/loggers/ExtendedLogChannel.h index b90494273f2..e70cd7b3094 100644 --- a/libs/libdaemon/include/daemon/ExtendedLogChannel.h +++ b/libs/libloggers/loggers/ExtendedLogChannel.h @@ -1,11 +1,13 @@ #pragma once -#include -#include +#include +namespace Poco +{ +class Message; +} namespace DB { - /// Poco::Message with more ClickHouse-specific info /// NOTE: Poco::Message is not polymorphic class, so we can't use inheritance in couple with dynamic_cast<>() class ExtendedLogMessage @@ -19,10 +21,10 @@ public: // Do not copy for efficiency reasons const Poco::Message & base; - UInt32 time_seconds = 0; - UInt32 time_microseconds = 0; + uint32_t time_seconds = 0; + uint32_t time_microseconds = 0; - UInt32 thread_number = 0; + uint32_t thread_number = 0; std::string query_id; }; diff --git a/libs/libloggers/loggers/Loggers.cpp b/libs/libloggers/loggers/Loggers.cpp new file mode 100644 index 00000000000..bc53cff27aa --- /dev/null +++ b/libs/libloggers/loggers/Loggers.cpp @@ -0,0 +1,164 @@ +#include "Loggers.h" + +#include +#include +#include +#include "OwnFormattingChannel.h" +#include "OwnPatternFormatter.h" +#include "OwnSplitChannel.h" +#include +#include +#include +#include +#include + +// TODO: move to libcommon +static std::string createDirectory(const std::string & file) +{ + auto path = Poco::Path(file).makeParent(); + if (path.toString().empty()) + return ""; + Poco::File(path).createDirectories(); + return path.toString(); +}; + +void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger /*_root*/, const std::string & cmd_name) +{ + auto current_logger = config.getString("logger", ""); + if (config_logger == current_logger) + return; + config_logger = current_logger; + + bool is_daemon = config.getBool("application.runAsDaemon", false); + + /// Split logs to ordinary log, error log, syslog and console. + /// Use extended interface of Channel for more comprehensive logging. + Poco::AutoPtr split = new DB::OwnSplitChannel; + + auto log_level = config.getString("logger.level", "trace"); + const auto log_path = config.getString("logger.log", ""); + if (!log_path.empty()) + { + createDirectory(log_path); + std::cerr << "Logging " << log_level << " to " << log_path << std::endl; + + // Set up two channel chains. + log_file = new Poco::FileChannel; + log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(log_path).absolute().toString()); + log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M")); + log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); + log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true")); + log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1")); + log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true")); + log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config.getRawString("logger.rotateOnOpen", "false")); + log_file->open(); + + Poco::AutoPtr pf = new OwnPatternFormatter(this); + + Poco::AutoPtr log = new DB::OwnFormattingChannel(pf, log_file); + split->addChannel(log); + } + + const auto errorlog_path = config.getString("logger.errorlog", ""); + if (!errorlog_path.empty()) + { + createDirectory(errorlog_path); + std::cerr << "Logging errors to " << errorlog_path << std::endl; + + error_log_file = new Poco::FileChannel; + error_log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(errorlog_path).absolute().toString()); + error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M")); + error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); + error_log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true")); + error_log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1")); + error_log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true")); + error_log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config.getRawString("logger.rotateOnOpen", "false")); + + Poco::AutoPtr pf = new OwnPatternFormatter(this); + + Poco::AutoPtr errorlog = new DB::OwnFormattingChannel(pf, error_log_file); + errorlog->setLevel(Poco::Message::PRIO_NOTICE); + errorlog->open(); + split->addChannel(errorlog); + } + + /// "dynamic_layer_selection" is needed only for Yandex.Metrika, that share part of ClickHouse code. + /// We don't need this configuration parameter. + + if (config.getBool("logger.use_syslog", false) || config.getBool("dynamic_layer_selection", false)) + { + //const std::string & cmd_name = commandName(); + + if (config.has("logger.syslog.address")) + { + syslog_channel = new Poco::Net::RemoteSyslogChannel(); + // syslog address + syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_LOGHOST, config.getString("logger.syslog.address")); + if (config.has("logger.syslog.hostname")) + { + syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_HOST, config.getString("logger.syslog.hostname")); + } + syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_FORMAT, config.getString("logger.syslog.format", "syslog")); + syslog_channel->setProperty( + Poco::Net::RemoteSyslogChannel::PROP_FACILITY, config.getString("logger.syslog.facility", "LOG_USER")); + } + else + { + syslog_channel = new Poco::SyslogChannel(); + syslog_channel->setProperty(Poco::SyslogChannel::PROP_NAME, cmd_name); + syslog_channel->setProperty(Poco::SyslogChannel::PROP_OPTIONS, config.getString("logger.syslog.options", "LOG_CONS|LOG_PID")); + syslog_channel->setProperty(Poco::SyslogChannel::PROP_FACILITY, config.getString("logger.syslog.facility", "LOG_DAEMON")); + } + syslog_channel->open(); + + Poco::AutoPtr pf = new OwnPatternFormatter(this, OwnPatternFormatter::ADD_LAYER_TAG); + + Poco::AutoPtr log = new DB::OwnFormattingChannel(pf, syslog_channel); + split->addChannel(log); + } + + if (config.getBool("logger.console", false) + || (!config.hasProperty("logger.console") && !is_daemon && (isatty(STDIN_FILENO) || isatty(STDERR_FILENO)))) + { + Poco::AutoPtr log = new DB::OwnFormattingChannel(new OwnPatternFormatter(this), new Poco::ConsoleChannel); + logger.warning("Logging " + log_level + " to console"); + split->addChannel(log); + } + + split->open(); + logger.close(); + logger.setChannel(split); + + // Global logging level (it can be overridden for specific loggers). + logger.setLevel(log_level); + + // Set level to all already created loggers + std::vector names; + //logger_root = Logger::root(); + logger.root().names(names); + for (const auto & name : names) + logger.root().get(name).setLevel(log_level); + + // Attach to the root logger. + logger.root().setLevel(log_level); + logger.root().setChannel(logger.getChannel()); + + // Explicitly specified log levels for specific loggers. + Poco::Util::AbstractConfiguration::Keys levels; + config.keys("logger.levels", levels); + + if (!levels.empty()) + for (const auto & level : levels) + logger.root().get(level).setLevel(config.getString("logger.levels." + level, "trace")); +} + +void Loggers::closeLogs(Poco::Logger & logger) +{ + if (log_file) + log_file->close(); + if (error_log_file) + error_log_file->close(); + + if (!log_file) + logger.warning("Logging to console but received signal to close log file (ignoring)."); +} diff --git a/libs/libloggers/loggers/Loggers.h b/libs/libloggers/loggers/Loggers.h new file mode 100644 index 00000000000..9090d5c3e7c --- /dev/null +++ b/libs/libloggers/loggers/Loggers.h @@ -0,0 +1,37 @@ +#include +#include +#include +#include +#include + +namespace Poco::Util +{ +class AbstractConfiguration; +} + + +class Loggers +{ +public: + /// Строит необходимые логгеры + void buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger, const std::string & cmd_name = ""); + + /// Закрыть файлы с логами. При следующей записи, будут созданы новые файлы. + void closeLogs(Poco::Logger & logger); + + std::optional getLayer() const + { + return layer; /// layer выставляется в классе-наследнике BaseDaemonApplication. + } + +protected: + std::optional layer; + +private: + /// Файлы с логами. + Poco::AutoPtr log_file; + Poco::AutoPtr error_log_file; + Poco::AutoPtr syslog_channel; + /// Previous value of logger element in config. It is used to reinitialize loggers whenever the value changed. + std::string config_logger; +}; diff --git a/libs/libdaemon/src/OwnFormattingChannel.cpp b/libs/libloggers/loggers/OwnFormattingChannel.cpp similarity index 87% rename from libs/libdaemon/src/OwnFormattingChannel.cpp rename to libs/libloggers/loggers/OwnFormattingChannel.cpp index a91f96bf650..9fccc929364 100644 --- a/libs/libdaemon/src/OwnFormattingChannel.cpp +++ b/libs/libloggers/loggers/OwnFormattingChannel.cpp @@ -1,10 +1,9 @@ -#include -#include +#include "OwnFormattingChannel.h" +#include "OwnPatternFormatter.h" namespace DB { - void OwnFormattingChannel::logExtended(const ExtendedLogMessage & msg) { if (pChannel && priority >= msg.base.getPriority()) diff --git a/libs/libdaemon/include/daemon/OwnFormattingChannel.h b/libs/libloggers/loggers/OwnFormattingChannel.h similarity index 70% rename from libs/libdaemon/include/daemon/OwnFormattingChannel.h rename to libs/libloggers/loggers/OwnFormattingChannel.h index 52045606c14..cd2e66279d7 100644 --- a/libs/libdaemon/include/daemon/OwnFormattingChannel.h +++ b/libs/libloggers/loggers/OwnFormattingChannel.h @@ -1,31 +1,26 @@ #pragma once -#include #include #include #include +#include "ExtendedLogChannel.h" #include "OwnPatternFormatter.h" namespace DB { - - // Like Poco::FormattingChannel but supports the extended logging interface and log level filter class OwnFormattingChannel : public Poco::Channel, public ExtendedLogChannel { public: - explicit OwnFormattingChannel(Poco::AutoPtr pFormatter_ = nullptr, Poco::AutoPtr pChannel_ = nullptr) - : pFormatter(std::move(pFormatter_)), pChannel(std::move(pChannel_)) {} - - void setChannel(Poco::AutoPtr pChannel_) + explicit OwnFormattingChannel( + Poco::AutoPtr pFormatter_ = nullptr, Poco::AutoPtr pChannel_ = nullptr) + : pFormatter(std::move(pFormatter_)), pChannel(std::move(pChannel_)) { - pChannel = std::move(pChannel_); } - void setLevel(Poco::Message::Priority priority_) - { - priority = priority_; - } + void setChannel(Poco::AutoPtr pChannel_) { pChannel = std::move(pChannel_); } + + void setLevel(Poco::Message::Priority priority_) { priority = priority_; } void open() override { diff --git a/libs/libdaemon/src/OwnPatternFormatter.cpp b/libs/libloggers/loggers/OwnPatternFormatter.cpp similarity index 91% rename from libs/libdaemon/src/OwnPatternFormatter.cpp rename to libs/libloggers/loggers/OwnPatternFormatter.cpp index 70f102ef351..ce4e8d10c92 100644 --- a/libs/libdaemon/src/OwnPatternFormatter.cpp +++ b/libs/libloggers/loggers/OwnPatternFormatter.cpp @@ -1,19 +1,20 @@ -#include - -#include -#include -#include -#include +#include "OwnPatternFormatter.h" #include #include +#include +#include +#include #include +#include #include -#include +#include "Loggers.h" -OwnPatternFormatter::OwnPatternFormatter(const BaseDaemon * daemon_, OwnPatternFormatter::Options options_) - : Poco::PatternFormatter(""), daemon(daemon_), options(options_) {} +OwnPatternFormatter::OwnPatternFormatter(const /*BaseDaemon*/ Loggers * daemon_, OwnPatternFormatter::Options options_) + : Poco::PatternFormatter(""), daemon(daemon_), options(options_) +{ +} void OwnPatternFormatter::formatExtended(const DB::ExtendedLogMessage & msg_ext, std::string & text) diff --git a/libs/libdaemon/include/daemon/OwnPatternFormatter.h b/libs/libloggers/loggers/OwnPatternFormatter.h similarity index 89% rename from libs/libdaemon/include/daemon/OwnPatternFormatter.h rename to libs/libloggers/loggers/OwnPatternFormatter.h index e90734c21e8..00a1b0a0313 100644 --- a/libs/libdaemon/include/daemon/OwnPatternFormatter.h +++ b/libs/libloggers/loggers/OwnPatternFormatter.h @@ -2,7 +2,7 @@ #include -#include +#include "ExtendedLogChannel.h" /** Форматирует по своему. @@ -19,12 +19,11 @@ * Также сделан чуть более эффективным (что имеет мало значения). */ -class BaseDaemon; +class Loggers; class OwnPatternFormatter : public Poco::PatternFormatter { public: - /// ADD_LAYER_TAG is needed only for Yandex.Metrika, that share part of ClickHouse code. enum Options { @@ -32,12 +31,12 @@ public: ADD_LAYER_TAG = 1 << 0 }; - OwnPatternFormatter(const BaseDaemon * daemon_, Options options_ = ADD_NOTHING); + OwnPatternFormatter(const Loggers * daemon_, Options options_ = ADD_NOTHING); void format(const Poco::Message & msg, std::string & text) override; void formatExtended(const DB::ExtendedLogMessage & msg_ext, std::string & text); private: - const BaseDaemon * daemon; + const Loggers * daemon; Options options; }; diff --git a/libs/libdaemon/src/OwnSplitChannel.cpp b/libs/libloggers/loggers/OwnSplitChannel.cpp similarity index 97% rename from libs/libdaemon/src/OwnSplitChannel.cpp rename to libs/libloggers/loggers/OwnSplitChannel.cpp index f7cb28764d3..0779f6477c3 100644 --- a/libs/libdaemon/src/OwnSplitChannel.cpp +++ b/libs/libloggers/loggers/OwnSplitChannel.cpp @@ -1,21 +1,17 @@ -#include +#include "OwnSplitChannel.h" +#include +#include +#include +#include +#include #include #include -#include -#include - -#include #include -#include -#include - namespace DB { - - void OwnSplitChannel::log(const Poco::Message & msg) { auto logs_queue = CurrentThread::getInternalTextLogsQueue(); diff --git a/libs/libdaemon/include/daemon/OwnSplitChannel.h b/libs/libloggers/loggers/OwnSplitChannel.h similarity index 95% rename from libs/libdaemon/include/daemon/OwnSplitChannel.h rename to libs/libloggers/loggers/OwnSplitChannel.h index ffdca4f460b..3579218f75c 100644 --- a/libs/libdaemon/include/daemon/OwnSplitChannel.h +++ b/libs/libloggers/loggers/OwnSplitChannel.h @@ -1,13 +1,12 @@ #pragma once -#include -#include -#include #include +#include +#include +#include "ExtendedLogChannel.h" namespace DB { - /// Works as Poco::SplitterChannel, but performs additional work: /// passes logs to Client via TCP interface /// tries to use extended logging interface of child for more comprehensive logging diff --git a/utils/build/build_macos.sh b/utils/build/build_macos.sh index 229e2d6937d..a9bf2481a0e 100755 --- a/utils/build/build_macos.sh +++ b/utils/build/build_macos.sh @@ -14,6 +14,10 @@ fi brew install cmake ninja gcc icu4c mariadb-connector-c openssl unixodbc libtool gettext readline librdkafka +# If you want to run tests +brew install python +sudo pip install lxml termcolor requests + ## Checkout ClickHouse sources # To get the latest stable version: