diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 014141aa33b..411bb42782f 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -1488,6 +1488,8 @@ Keys: - `formatting` – Log format for console output. Currently, only `json` is supported). - `use_syslog` - Also forward log output to syslog. - `syslog_level` - Log level for logging to syslog. +- `message_regexp` - Only log messages that match this regular expression. Defaults to `""`, indicating no filtering. +- `message_regexp_negative` - Only log messages that don't match this regular expression. Defaults to `""`, indicating no filtering. **Log format specifiers** @@ -1576,6 +1578,28 @@ The log level of individual log names can be overridden. For example, to mute al ``` +**Regular Expression Filtering** + +The messages logged can be filtered using regular expressions using `message_regexp` and `message_regexp_negative`. This can be done on a per-level basis or globally. If both are specified for a particular logger, the global expression is ignored and the per-level one overrides it. + + +```xml + + + .*executeQuery.* + .*ConfigReloader.* + + + + RBAC + + .*Application.* + .*Setting.* + + + +``` + ### syslog To write log messages additionally to syslog: diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 184f147a86a..ad665b5df35 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -611,10 +612,14 @@ void LocalServer::processConfig() if (getClientConfiguration().has("server_logs_file")) { + std::string pos_pattern = getClientConfiguration().getRawString("logger.message_regexp", ""); + std::string neg_pattern = getClientConfiguration().getRawString("logger.message_regexp_negative", ""); + Poco::AutoPtr filter_channel = new OwnFilteringChannel(new Poco::SimpleFileChannel(server_logs_file), nullptr, pos_pattern, neg_pattern); + auto poco_logs_level = Poco::Logger::parseLevel(level); Poco::Logger::root().setLevel(poco_logs_level); Poco::AutoPtr pf = new OwnPatternFormatter; - Poco::AutoPtr log = new OwnFormattingChannel(pf, new Poco::SimpleFileChannel(server_logs_file)); + Poco::AutoPtr log = new OwnFormattingChannel(pf, filter_channel); Poco::Logger::root().setChannel(log); } else diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index d4d3ad58ddd..be53198119e 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -1,3 +1,4 @@ +#include "Loggers/OwnFilteringChannel.h" #pragma clang diagnostic ignored "-Wreserved-identifier" #include @@ -625,7 +626,12 @@ void BaseDaemon::setupWatchdog() pf = new OwnJSONPatternFormatter(config()); else pf = new OwnPatternFormatter; - Poco::AutoPtr log = new OwnFormattingChannel(pf, new Poco::ConsoleChannel(std::cerr)); + + // Apply regexp filtering after receiving the formatting channel + std::string pos_pattern = config().getRawString("logger.message_regexp", ""); + std::string neg_pattern = config().getRawString("logger.message_regexp_negative", ""); + Poco::AutoPtr filter_channel = new OwnFilteringChannel(new Poco::ConsoleChannel(std::cerr), nullptr, pos_pattern, neg_pattern); + Poco::AutoPtr log = new OwnFormattingChannel(pf, filter_channel); logger().setChannel(log); } diff --git a/src/Loggers/Loggers.cpp b/src/Loggers/Loggers.cpp index 35b96bce42a..e0cfb018505 100644 --- a/src/Loggers/Loggers.cpp +++ b/src/Loggers/Loggers.cpp @@ -1,5 +1,6 @@ #include "Loggers.h" +#include "Loggers/OwnFilteringChannel.h" #include "OwnFormattingChannel.h" #include "OwnPatternFormatter.h" #include "OwnSplitChannel.h" @@ -12,6 +13,7 @@ #include #include #include +#include "Common/Exception.h" #ifndef WITHOUT_TEXT_LOG #include @@ -28,6 +30,7 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int TYPE_MISMATCH; } } @@ -221,7 +224,17 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log split->open(); logger.close(); - logger.setChannel(split); + std::string global_pos_pattern = config.getRawString("logger.message_regexp", ""); + std::string global_neg_pattern = config.getRawString("logger.message_regexp_negative", ""); + + Poco::AutoPtr pf; + if (config.getString("logger.formatting.type", "") == "json") + pf = new OwnJSONPatternFormatter(config); + else + pf = new OwnPatternFormatter; + + Poco::AutoPtr filter_channel = new DB::OwnFilteringChannel(split, pf, global_pos_pattern, global_neg_pattern); + logger.setChannel(filter_channel); logger.setLevel(max_log_level); // Global logging level and channel (it can be overridden for specific loggers). @@ -235,7 +248,10 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log for (const auto & name : names) { logger.get(name).setLevel(max_log_level); - logger.get(name).setChannel(split); + + // Create a new filter channel for each logger that share the same split channel + filter_channel = new DB::OwnFilteringChannel(split, pf, global_pos_pattern, global_neg_pattern); + logger.get(name).setChannel(filter_channel); } // Explicitly specified log levels for specific loggers. @@ -251,6 +267,15 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log { const std::string name(config.getString("logger.levels." + key + ".name")); const std::string level(config.getString("logger.levels." + key + ".level")); + + std::string pos_pattern = config.getRawString("logger.levels." + key + "message_regexp", ""); + std::string neg_pattern = config.getRawString("logger.levels." + key + "message_regexp_negative", ""); + + if (auto * regexp_channel = dynamic_cast(logger.root().get(name).getChannel())) + regexp_channel->setRegexpPatterns(pos_pattern, neg_pattern); + else + throw DB::Exception(DB::ErrorCodes::TYPE_MISMATCH, "Couldn't convert to OwnFilteringChannel."); + logger.root().get(name).setLevel(level); } else @@ -347,16 +372,29 @@ void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Log } split->setLevel("syslog", syslog_level); + std::string global_pos_pattern = config.getRawString("logger.message_regexp", ""); + std::string global_neg_pattern = config.getRawString("logger.message_regexp_negative", ""); + // Global logging level (it can be overridden for specific loggers). logger.setLevel(max_log_level); + if (auto * regexp_channel = dynamic_cast(logger.getChannel())) + regexp_channel->setRegexpPatterns(global_pos_pattern, global_neg_pattern); + else + throw DB::Exception(DB::ErrorCodes::TYPE_MISMATCH, "Couldn't convert to OwnFilteringChannel."); // Set level to all already created loggers std::vector names; - logger.root().names(names); + + // Set all to global in case logger.levels are not specified for (const auto & name : names) + { logger.root().get(name).setLevel(max_log_level); + if (auto * regexp_channel = dynamic_cast(logger.root().get(name).getChannel())) + regexp_channel->setRegexpPatterns(global_pos_pattern, global_neg_pattern); + } + logger.root().setLevel(max_log_level); // Explicitly specified log levels for specific loggers. @@ -373,6 +411,14 @@ void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Log const std::string name(config.getString("logger.levels." + key + ".name")); const std::string level(config.getString("logger.levels." + key + ".level")); logger.root().get(name).setLevel(level); + + std::string pos_pattern = config.getRawString("logger.levels." + key + "message_regexp", global_pos_pattern); + std::string neg_pattern = config.getRawString("logger.levels." + key + "message_regexp_negative", global_neg_pattern); + + if (auto * regexp_channel = dynamic_cast(logger.root().get(name).getChannel())) + regexp_channel->setRegexpPatterns(pos_pattern, neg_pattern); + else + throw DB::Exception(DB::ErrorCodes::TYPE_MISMATCH, "Couldn't convert to OwnFilteringChannel."); } else { diff --git a/src/Loggers/OwnFilteringChannel.cpp b/src/Loggers/OwnFilteringChannel.cpp new file mode 100644 index 00000000000..850de858a5f --- /dev/null +++ b/src/Loggers/OwnFilteringChannel.cpp @@ -0,0 +1,47 @@ +#include "OwnFilteringChannel.h" +#include + + +namespace DB +{ + +void OwnFilteringChannel::log(const Poco::Message & msg) +{ + std::string formatted_text; + + // Apply formatting to the text + if (pFormatter) + { + pFormatter->formatExtended(ExtendedLogMessage::getFrom(msg), formatted_text); + } + else + { + formatted_text = msg.getText(); + } + if (!regexpFilteredOut(formatted_text)) + pChannel->log(msg); +} + +bool OwnFilteringChannel::regexpFilteredOut(std::string text) const +{ + if (!positive_pattern.empty()) + { + Poco::RegularExpression positive_regexp(positive_pattern); + if (!positive_regexp.match(text)) + { + return true; + } + } + + if (!negative_pattern.empty()) + { + Poco::RegularExpression negative_regexp(negative_pattern); + if (negative_regexp.match(text)) + { + return true; + } + } + return false; +} + +} diff --git a/src/Loggers/OwnFilteringChannel.h b/src/Loggers/OwnFilteringChannel.h new file mode 100644 index 00000000000..0d8cff493a0 --- /dev/null +++ b/src/Loggers/OwnFilteringChannel.h @@ -0,0 +1,68 @@ +#pragma once +#include +#include +#include +#include +#include "OwnPatternFormatter.h" + + +namespace DB +{ + +// Filters the logs based on regular expressions. Should be processed after formatting channel to read entire formatted text +class OwnFilteringChannel : public Poco::Channel +{ +public: + explicit OwnFilteringChannel(Poco::AutoPtr pChannel_, Poco::AutoPtr pf, + std::string positive_pattern_, std::string negative_pattern_) + : positive_pattern(positive_pattern_), negative_pattern(negative_pattern_), pChannel(pChannel_), pFormatter(pf) + { + } + + // Only log if pass both positive and negative regexp checks. + // Checks the regexps on the formatted text (without color), but then passes the raw text + // to the split channel to handle formatting for individual channels (e.g apply color) + void log(const Poco::Message & msg) override; + + // Sets the regex patterns to use for filtering. Specifying an empty string pattern "" indicates no filtering + void setRegexpPatterns(std::string positive_pattern_, std::string negative_pattern_) + { + positive_pattern = positive_pattern_; + negative_pattern = negative_pattern_; + } + + void open() override + { + if (pChannel) + pChannel->open(); + } + + void close() override + { + if (pChannel) + pChannel->close(); + } + + void setProperty(const std::string& name, const std::string& value) override + { + if (pChannel) + pChannel->setProperty(name, value); + } + + std::string getProperty(const std::string& name) const override + { + if (pChannel) + return pChannel->getProperty(name); + return ""; + } + +private: + bool regexpFilteredOut(std::string text) const; + + std::string positive_pattern; + std::string negative_pattern; + Poco::AutoPtr pChannel; + Poco::AutoPtr pFormatter; +}; + +}