Merge 2360dc0053 into 3eb5bc1a0f

docs/en/operations/server-configuration-parameters/settings.md

@@ -1488,6 +1488,8 @@ Keys:
 - `formatting` – Log format for console output. Currently, only `json` is supported).
 - `use_syslog` - Also forward log output to syslog.
 - `syslog_level` - Log level for logging to syslog.
+- `message_regexp` - Only log messages that match this regular expression. Defaults to `""`, indicating no filtering.
+- `message_regexp_negative` - Only log messages that don't match this regular expression. Defaults to `""`, indicating no filtering.
 
 **Log format specifiers**
 
@@ -1576,6 +1578,28 @@ The log level of individual log names can be overridden. For example, to mute al
 </logger>
 ```
 
+**Regular Expression Filtering**
+
+The messages logged can be filtered using regular expressions using `message_regexp` and `message_regexp_negative`. This can be done on a per-level basis or globally. If both are specified for a particular logger, the global expression is ignored and the per-level one overrides it.
+
+
+```xml
+<logger>
+    <!-- Global: Only log messages that have 'executeQuery' in them and not 'ConfigReloader' -->
+    <message_regexp>.*executeQuery.*</message_regexp>
+    <message_regexp>.*ConfigReloader.*</message_regexp>
+
+    <levels>
+        <logger>
+            <name>RBAC</name>
+            <!-- For logger 'RBAC', instead of matching for '.*executeQuery.*' and '.*ConfigReloader.*' match instead for '.*Application.*' and '.*Setting.*'. -->
+            <message_regexp>.*Application.*</message_regexp>
+            <message_regexp_negative>.*Setting.*</message_regexp_negative>
+        </logger>
+    </levels>
+</logger>
+```
+
 ### syslog
 
 To write log messages additionally to syslog:

programs/local/LocalServer.cpp

@@ -34,6 +34,7 @@
 #include <Common/randomSeed.h>
 #include <Common/ThreadPool.h>
 #include <Common/CurrentMetrics.h>
+#include <Loggers/OwnFilteringChannel.h>
 #include <Loggers/OwnFormattingChannel.h>
 #include <Loggers/OwnPatternFormatter.h>
 #include <IO/ReadBufferFromFile.h>
@@ -611,10 +612,14 @@ void LocalServer::processConfig()
 
     if (getClientConfiguration().has("server_logs_file"))
     {
+        std::string pos_pattern = getClientConfiguration().getRawString("logger.message_regexp", "");
+        std::string neg_pattern = getClientConfiguration().getRawString("logger.message_regexp_negative", "");
+        Poco::AutoPtr<OwnFilteringChannel> filter_channel = new OwnFilteringChannel(new Poco::SimpleFileChannel(server_logs_file), nullptr, pos_pattern, neg_pattern);
+
         auto poco_logs_level = Poco::Logger::parseLevel(level);
         Poco::Logger::root().setLevel(poco_logs_level);
         Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter;
-        Poco::AutoPtr<OwnFormattingChannel> log = new OwnFormattingChannel(pf, new Poco::SimpleFileChannel(server_logs_file));
+        Poco::AutoPtr<OwnFormattingChannel> log = new OwnFormattingChannel(pf, filter_channel);
         Poco::Logger::root().setChannel(log);
     }
     else

src/Daemon/BaseDaemon.cpp

@@ -1,3 +1,4 @@
+#include "Loggers/OwnFilteringChannel.h"
 #pragma clang diagnostic ignored "-Wreserved-identifier"
 
 #include <base/defines.h>
@@ -625,7 +626,12 @@ void BaseDaemon::setupWatchdog()
                 pf = new OwnJSONPatternFormatter(config());
             else
                 pf = new OwnPatternFormatter;
-            Poco::AutoPtr<OwnFormattingChannel> log = new OwnFormattingChannel(pf, new Poco::ConsoleChannel(std::cerr));
+
+            // Apply regexp filtering after receiving the formatting channel
+            std::string pos_pattern = config().getRawString("logger.message_regexp", "");
+            std::string neg_pattern = config().getRawString("logger.message_regexp_negative", "");
+            Poco::AutoPtr<OwnFilteringChannel> filter_channel = new OwnFilteringChannel(new Poco::ConsoleChannel(std::cerr), nullptr, pos_pattern, neg_pattern);
+            Poco::AutoPtr<OwnFormattingChannel> log = new OwnFormattingChannel(pf, filter_channel);
             logger().setChannel(log);
         }
 

src/Loggers/Loggers.cpp

@@ -1,5 +1,6 @@
 #include "Loggers.h"
 
+#include "Loggers/OwnFilteringChannel.h"
 #include "OwnFormattingChannel.h"
 #include "OwnPatternFormatter.h"
 #include "OwnSplitChannel.h"
@@ -12,6 +13,7 @@
 #include <Poco/Net/RemoteSyslogChannel.h>
 #include <Poco/SyslogChannel.h>
 #include <Poco/Util/AbstractConfiguration.h>
+#include "Common/Exception.h"
 
 #ifndef WITHOUT_TEXT_LOG
     #include <Interpreters/TextLog.h>
@@ -28,6 +30,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int BAD_ARGUMENTS;
+    extern const int TYPE_MISMATCH;
 }
 
 }
@@ -221,7 +224,17 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
     split->open();
     logger.close();
 
-    logger.setChannel(split);
+    std::string global_pos_pattern = config.getRawString("logger.message_regexp", "");
+    std::string global_neg_pattern = config.getRawString("logger.message_regexp_negative", "");
+
+    Poco::AutoPtr<OwnPatternFormatter> pf;
+    if (config.getString("logger.formatting.type", "") == "json")
+        pf = new OwnJSONPatternFormatter(config);
+    else
+        pf = new OwnPatternFormatter;
+
+    Poco::AutoPtr<DB::OwnFilteringChannel> filter_channel = new DB::OwnFilteringChannel(split, pf, global_pos_pattern, global_neg_pattern);
+    logger.setChannel(filter_channel);
     logger.setLevel(max_log_level);
 
     // Global logging level and channel (it can be overridden for specific loggers).
@@ -235,7 +248,10 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
     for (const auto & name : names)
     {
         logger.get(name).setLevel(max_log_level);
-        logger.get(name).setChannel(split);
+
+        // Create a new filter channel for each logger that share the same split channel
+        filter_channel = new DB::OwnFilteringChannel(split, pf, global_pos_pattern, global_neg_pattern);
+        logger.get(name).setChannel(filter_channel);
     }
 
     // Explicitly specified log levels for specific loggers.
@@ -251,6 +267,15 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
                 {
                     const std::string name(config.getString("logger.levels." + key + ".name"));
                     const std::string level(config.getString("logger.levels." + key + ".level"));
+
+                    std::string pos_pattern = config.getRawString("logger.levels." + key + "message_regexp", "");
+                    std::string neg_pattern = config.getRawString("logger.levels." + key + "message_regexp_negative", "");
+
+                    if (auto * regexp_channel = dynamic_cast<DB::OwnFilteringChannel*>(logger.root().get(name).getChannel()))
+                        regexp_channel->setRegexpPatterns(pos_pattern, neg_pattern);
+                    else
+                        throw DB::Exception(DB::ErrorCodes::TYPE_MISMATCH, "Couldn't convert to OwnFilteringChannel.");
+
                     logger.root().get(name).setLevel(level);
                 }
                 else
@@ -347,16 +372,29 @@ void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Log
     }
     split->setLevel("syslog", syslog_level);
 
+    std::string global_pos_pattern = config.getRawString("logger.message_regexp", "");
+    std::string global_neg_pattern = config.getRawString("logger.message_regexp_negative", "");
+
     // Global logging level (it can be overridden for specific loggers).
     logger.setLevel(max_log_level);
+    if (auto * regexp_channel = dynamic_cast<DB::OwnFilteringChannel*>(logger.getChannel()))
+        regexp_channel->setRegexpPatterns(global_pos_pattern, global_neg_pattern);
+    else
+        throw DB::Exception(DB::ErrorCodes::TYPE_MISMATCH, "Couldn't convert to OwnFilteringChannel.");
 
     // Set level to all already created loggers
     std::vector<std::string> names;
-
     logger.root().names(names);
+
+    // Set all to global in case logger.levels are not specified
     for (const auto & name : names)
+    {
         logger.root().get(name).setLevel(max_log_level);
 
+        if (auto * regexp_channel = dynamic_cast<DB::OwnFilteringChannel*>(logger.root().get(name).getChannel()))
+            regexp_channel->setRegexpPatterns(global_pos_pattern, global_neg_pattern);
+    }
+
     logger.root().setLevel(max_log_level);
 
     // Explicitly specified log levels for specific loggers.
@@ -373,6 +411,14 @@ void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Log
                     const std::string name(config.getString("logger.levels." + key + ".name"));
                     const std::string level(config.getString("logger.levels." + key + ".level"));
                     logger.root().get(name).setLevel(level);
+
+                    std::string pos_pattern = config.getRawString("logger.levels." + key + "message_regexp", global_pos_pattern);
+                    std::string neg_pattern = config.getRawString("logger.levels." + key + "message_regexp_negative", global_neg_pattern);
+
+                    if (auto * regexp_channel = dynamic_cast<DB::OwnFilteringChannel*>(logger.root().get(name).getChannel()))
+                        regexp_channel->setRegexpPatterns(pos_pattern, neg_pattern);
+                    else
+                        throw DB::Exception(DB::ErrorCodes::TYPE_MISMATCH, "Couldn't convert to OwnFilteringChannel.");
                 }
                 else
                 {

src/Loggers/OwnFilteringChannel.cpp

@@ -0,0 +1,47 @@
+#include "OwnFilteringChannel.h"
+#include <Poco/RegularExpression.h>
+
+
+namespace DB
+{
+
+void OwnFilteringChannel::log(const Poco::Message & msg)
+{
+    std::string formatted_text;
+
+    // Apply formatting to the text
+    if (pFormatter)
+    {
+        pFormatter->formatExtended(ExtendedLogMessage::getFrom(msg), formatted_text);
+    }
+    else
+    {
+        formatted_text = msg.getText();
+    }
+    if (!regexpFilteredOut(formatted_text))
+        pChannel->log(msg);
+}
+
+bool OwnFilteringChannel::regexpFilteredOut(std::string text) const
+{
+    if (!positive_pattern.empty())
+    {
+        Poco::RegularExpression positive_regexp(positive_pattern);
+        if (!positive_regexp.match(text))
+        {
+            return true;
+        }
+    }
+
+    if (!negative_pattern.empty())
+    {
+        Poco::RegularExpression negative_regexp(negative_pattern);
+        if (negative_regexp.match(text))
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
+}

src/Loggers/OwnFilteringChannel.h

@@ -0,0 +1,68 @@
+#pragma once
+#include <Poco/AutoPtr.h>
+#include <Poco/Channel.h>
+#include <Poco/Message.h>
+#include <Poco/Util/AbstractConfiguration.h>
+#include "OwnPatternFormatter.h"
+
+
+namespace DB
+{
+
+// Filters the logs based on regular expressions. Should be processed after formatting channel to read entire formatted text
+class OwnFilteringChannel : public Poco::Channel
+{
+public:
+    explicit OwnFilteringChannel(Poco::AutoPtr<Poco::Channel> pChannel_, Poco::AutoPtr<OwnPatternFormatter> pf,
+        std::string positive_pattern_, std::string negative_pattern_)
+    : positive_pattern(positive_pattern_), negative_pattern(negative_pattern_), pChannel(pChannel_), pFormatter(pf)
+    {
+    }
+
+    // Only log if pass both positive and negative regexp checks.
+    // Checks the regexps on the formatted text (without color), but then passes the raw text
+    // to the split channel to handle formatting for individual channels (e.g apply color)
+    void log(const Poco::Message & msg) override;
+
+    // Sets the regex patterns to use for filtering. Specifying an empty string pattern "" indicates no filtering
+    void setRegexpPatterns(std::string positive_pattern_, std::string negative_pattern_)
+    {
+        positive_pattern = positive_pattern_;
+        negative_pattern = negative_pattern_;
+    }
+
+    void open() override
+    {
+        if (pChannel)
+            pChannel->open();
+    }
+
+    void close() override
+    {
+        if (pChannel)
+            pChannel->close();
+    }
+
+    void setProperty(const std::string& name, const std::string& value) override
+    {
+        if (pChannel)
+            pChannel->setProperty(name, value);
+    }
+
+    std::string getProperty(const std::string& name) const override
+    {
+        if (pChannel)
+            return pChannel->getProperty(name);
+        return "";
+    }
+
+private:
+    bool regexpFilteredOut(std::string text) const;
+
+    std::string positive_pattern;
+    std::string negative_pattern;
+    Poco::AutoPtr<Poco::Channel> pChannel;
+    Poco::AutoPtr<OwnPatternFormatter> pFormatter;
+};
+
+}