diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md
index 014141aa33b..411bb42782f 100644
--- a/docs/en/operations/server-configuration-parameters/settings.md
+++ b/docs/en/operations/server-configuration-parameters/settings.md
@@ -1488,6 +1488,8 @@ Keys:
- `formatting` – Log format for console output. Currently, only `json` is supported).
- `use_syslog` - Also forward log output to syslog.
- `syslog_level` - Log level for logging to syslog.
+- `message_regexp` - Only log messages that match this regular expression. Defaults to `""`, indicating no filtering.
+- `message_regexp_negative` - Only log messages that don't match this regular expression. Defaults to `""`, indicating no filtering.
**Log format specifiers**
@@ -1576,6 +1578,28 @@ The log level of individual log names can be overridden. For example, to mute al
```
+**Regular Expression Filtering**
+
+The messages logged can be filtered using regular expressions using `message_regexp` and `message_regexp_negative`. This can be done on a per-level basis or globally. If both are specified for a particular logger, the global expression is ignored and the per-level one overrides it.
+
+
+```xml
+
+
+ .*executeQuery.*
+ .*ConfigReloader.*
+
+
+
+ RBAC
+
+ .*Application.*
+ .*Setting.*
+
+
+
+```
+
### syslog
To write log messages additionally to syslog:
diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index 184f147a86a..ad665b5df35 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -34,6 +34,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -611,10 +612,14 @@ void LocalServer::processConfig()
if (getClientConfiguration().has("server_logs_file"))
{
+ std::string pos_pattern = getClientConfiguration().getRawString("logger.message_regexp", "");
+ std::string neg_pattern = getClientConfiguration().getRawString("logger.message_regexp_negative", "");
+ Poco::AutoPtr filter_channel = new OwnFilteringChannel(new Poco::SimpleFileChannel(server_logs_file), nullptr, pos_pattern, neg_pattern);
+
auto poco_logs_level = Poco::Logger::parseLevel(level);
Poco::Logger::root().setLevel(poco_logs_level);
Poco::AutoPtr pf = new OwnPatternFormatter;
- Poco::AutoPtr log = new OwnFormattingChannel(pf, new Poco::SimpleFileChannel(server_logs_file));
+ Poco::AutoPtr log = new OwnFormattingChannel(pf, filter_channel);
Poco::Logger::root().setChannel(log);
}
else
diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp
index d4d3ad58ddd..be53198119e 100644
--- a/src/Daemon/BaseDaemon.cpp
+++ b/src/Daemon/BaseDaemon.cpp
@@ -1,3 +1,4 @@
+#include "Loggers/OwnFilteringChannel.h"
#pragma clang diagnostic ignored "-Wreserved-identifier"
#include
@@ -625,7 +626,12 @@ void BaseDaemon::setupWatchdog()
pf = new OwnJSONPatternFormatter(config());
else
pf = new OwnPatternFormatter;
- Poco::AutoPtr log = new OwnFormattingChannel(pf, new Poco::ConsoleChannel(std::cerr));
+
+ // Apply regexp filtering after receiving the formatting channel
+ std::string pos_pattern = config().getRawString("logger.message_regexp", "");
+ std::string neg_pattern = config().getRawString("logger.message_regexp_negative", "");
+ Poco::AutoPtr filter_channel = new OwnFilteringChannel(new Poco::ConsoleChannel(std::cerr), nullptr, pos_pattern, neg_pattern);
+ Poco::AutoPtr log = new OwnFormattingChannel(pf, filter_channel);
logger().setChannel(log);
}
diff --git a/src/Loggers/Loggers.cpp b/src/Loggers/Loggers.cpp
index 35b96bce42a..e0cfb018505 100644
--- a/src/Loggers/Loggers.cpp
+++ b/src/Loggers/Loggers.cpp
@@ -1,5 +1,6 @@
#include "Loggers.h"
+#include "Loggers/OwnFilteringChannel.h"
#include "OwnFormattingChannel.h"
#include "OwnPatternFormatter.h"
#include "OwnSplitChannel.h"
@@ -12,6 +13,7 @@
#include
#include
#include
+#include "Common/Exception.h"
#ifndef WITHOUT_TEXT_LOG
#include
@@ -28,6 +30,7 @@ namespace DB
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
+ extern const int TYPE_MISMATCH;
}
}
@@ -221,7 +224,17 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
split->open();
logger.close();
- logger.setChannel(split);
+ std::string global_pos_pattern = config.getRawString("logger.message_regexp", "");
+ std::string global_neg_pattern = config.getRawString("logger.message_regexp_negative", "");
+
+ Poco::AutoPtr pf;
+ if (config.getString("logger.formatting.type", "") == "json")
+ pf = new OwnJSONPatternFormatter(config);
+ else
+ pf = new OwnPatternFormatter;
+
+ Poco::AutoPtr filter_channel = new DB::OwnFilteringChannel(split, pf, global_pos_pattern, global_neg_pattern);
+ logger.setChannel(filter_channel);
logger.setLevel(max_log_level);
// Global logging level and channel (it can be overridden for specific loggers).
@@ -235,7 +248,10 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
for (const auto & name : names)
{
logger.get(name).setLevel(max_log_level);
- logger.get(name).setChannel(split);
+
+ // Create a new filter channel for each logger that share the same split channel
+ filter_channel = new DB::OwnFilteringChannel(split, pf, global_pos_pattern, global_neg_pattern);
+ logger.get(name).setChannel(filter_channel);
}
// Explicitly specified log levels for specific loggers.
@@ -251,6 +267,15 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
{
const std::string name(config.getString("logger.levels." + key + ".name"));
const std::string level(config.getString("logger.levels." + key + ".level"));
+
+ std::string pos_pattern = config.getRawString("logger.levels." + key + "message_regexp", "");
+ std::string neg_pattern = config.getRawString("logger.levels." + key + "message_regexp_negative", "");
+
+ if (auto * regexp_channel = dynamic_cast(logger.root().get(name).getChannel()))
+ regexp_channel->setRegexpPatterns(pos_pattern, neg_pattern);
+ else
+ throw DB::Exception(DB::ErrorCodes::TYPE_MISMATCH, "Couldn't convert to OwnFilteringChannel.");
+
logger.root().get(name).setLevel(level);
}
else
@@ -347,16 +372,29 @@ void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Log
}
split->setLevel("syslog", syslog_level);
+ std::string global_pos_pattern = config.getRawString("logger.message_regexp", "");
+ std::string global_neg_pattern = config.getRawString("logger.message_regexp_negative", "");
+
// Global logging level (it can be overridden for specific loggers).
logger.setLevel(max_log_level);
+ if (auto * regexp_channel = dynamic_cast(logger.getChannel()))
+ regexp_channel->setRegexpPatterns(global_pos_pattern, global_neg_pattern);
+ else
+ throw DB::Exception(DB::ErrorCodes::TYPE_MISMATCH, "Couldn't convert to OwnFilteringChannel.");
// Set level to all already created loggers
std::vector names;
-
logger.root().names(names);
+
+ // Set all to global in case logger.levels are not specified
for (const auto & name : names)
+ {
logger.root().get(name).setLevel(max_log_level);
+ if (auto * regexp_channel = dynamic_cast(logger.root().get(name).getChannel()))
+ regexp_channel->setRegexpPatterns(global_pos_pattern, global_neg_pattern);
+ }
+
logger.root().setLevel(max_log_level);
// Explicitly specified log levels for specific loggers.
@@ -373,6 +411,14 @@ void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Log
const std::string name(config.getString("logger.levels." + key + ".name"));
const std::string level(config.getString("logger.levels." + key + ".level"));
logger.root().get(name).setLevel(level);
+
+ std::string pos_pattern = config.getRawString("logger.levels." + key + "message_regexp", global_pos_pattern);
+ std::string neg_pattern = config.getRawString("logger.levels." + key + "message_regexp_negative", global_neg_pattern);
+
+ if (auto * regexp_channel = dynamic_cast(logger.root().get(name).getChannel()))
+ regexp_channel->setRegexpPatterns(pos_pattern, neg_pattern);
+ else
+ throw DB::Exception(DB::ErrorCodes::TYPE_MISMATCH, "Couldn't convert to OwnFilteringChannel.");
}
else
{
diff --git a/src/Loggers/OwnFilteringChannel.cpp b/src/Loggers/OwnFilteringChannel.cpp
new file mode 100644
index 00000000000..5feaf9af084
--- /dev/null
+++ b/src/Loggers/OwnFilteringChannel.cpp
@@ -0,0 +1,51 @@
+#include "OwnFilteringChannel.h"
+#include
+// #include // TODO
+
+
+namespace DB
+{
+
+void OwnFilteringChannel::log(const Poco::Message & msg)
+{
+ std::string formatted_text;
+
+ // Apply formatting to the text
+ if (pFormatter)
+ {
+ pFormatter->formatExtended(ExtendedLogMessage::getFrom(msg), formatted_text);
+ }
+ else
+ {
+ formatted_text = msg.getText();
+ }
+ if (!regexpFilteredOut(formatted_text))
+ pChannel->log(msg);
+}
+
+bool OwnFilteringChannel::regexpFilteredOut(std::string text) const
+{
+ if (!positive_pattern.empty())
+ {
+ Poco::RegularExpression positive_regexp(positive_pattern);
+ if (!positive_regexp.match(text))
+ {
+ // std::cout << "Skipping Message: " << text << "| due to positive regexp: " << positive_pattern << std::endl;
+ return true;
+ }
+ }
+
+ if (!negative_pattern.empty())
+ {
+ Poco::RegularExpression negative_regexp(negative_pattern);
+ if (negative_regexp.match(text))
+ {
+ // std::cout << "Skipping Message: " << text << "| due to negative regexp: " << negative_pattern << std::endl;
+ return true;
+ }
+ }
+ // std::cout << "THE FOLLOWING MESSAGE PASSED using positive: " << positive_pattern << " and negative: " << negative_pattern << std::endl;
+ return false;
+}
+
+}
diff --git a/src/Loggers/OwnFilteringChannel.h b/src/Loggers/OwnFilteringChannel.h
new file mode 100644
index 00000000000..74ee57a8419
--- /dev/null
+++ b/src/Loggers/OwnFilteringChannel.h
@@ -0,0 +1,61 @@
+#pragma once
+#include
+#include
+#include
+#include
+#include "OwnPatternFormatter.h"
+
+
+namespace DB
+{
+
+// Filters the logs based on regular expressions. Should be processed after formatting channel to read entire formatted text
+class OwnFilteringChannel : public Poco::Channel
+{
+public:
+ explicit OwnFilteringChannel(Poco::AutoPtr pChannel_, Poco::AutoPtr pf,
+ std::string positive_pattern_, std::string negative_pattern_)
+ : positive_pattern(positive_pattern_), negative_pattern(negative_pattern_), pChannel(pChannel_), pFormatter(pf)
+ {
+ }
+
+ // Only log if pass both positive and negative regexp checks.
+ // Checks the regexps on the formatted text (without color), but then passes the raw text
+ // to the split channel to handle formatting for individual channels (e.g apply color)
+ void log(const Poco::Message & msg) override;
+
+ // Sets the regex patterns to use for filtering. Specifying an empty string pattern "" indicates no filtering
+ void setRegexpPatterns(std::string positive_pattern_, std::string negative_pattern_)
+ {
+ positive_pattern = positive_pattern_;
+ negative_pattern = negative_pattern_;
+ }
+
+ void open() override
+ {
+ if (pChannel)
+ pChannel->open();
+ }
+
+ void close() override
+ {
+ if (pChannel)
+ pChannel->close();
+ }
+
+ void setProperty(const std::string& name, const std::string& value) override
+ {
+ if (pChannel)
+ pChannel->setProperty(name, value);
+ }
+
+private:
+ bool regexpFilteredOut(std::string text) const;
+
+ std::string positive_pattern;
+ std::string negative_pattern;
+ Poco::AutoPtr pChannel;
+ Poco::AutoPtr pFormatter;
+};
+
+}