From 90fa4863f9beabe094474e4d8516ddc1b0eac518 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 13 Jul 2022 16:15:37 +0800 Subject: [PATCH] Text log source regexp --- src/Core/Settings.h | 1 + src/Interpreters/InternalTextLogsQueue.cpp | 15 +++++++++++++++ src/Interpreters/InternalTextLogsQueue.h | 9 ++++++++- src/Loggers/OwnSplitChannel.cpp | 4 ++-- src/Server/GRPCServer.cpp | 1 + src/Server/TCPHandler.cpp | 1 + .../02359_send_logs_source_regexp.reference | 1 + .../0_stateless/02359_send_logs_source_regexp.sh | 11 +++++++++++ 8 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/02359_send_logs_source_regexp.reference create mode 100755 tests/queries/0_stateless/02359_send_logs_source_regexp.sh diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 5597d9076a4..0257c8e01e4 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -391,6 +391,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, log_query_views, true, "Log query dependent views into system.query_views_log table. This setting have effect only when 'log_queries' is true.", 0) \ M(String, log_comment, "", "Log comment into system.query_log table and server log. It can be set to arbitrary string no longer than max_query_size.", 0) \ M(LogsLevel, send_logs_level, LogsLevel::fatal, "Send server text logs with specified minimum level to client. Valid values: 'trace', 'debug', 'information', 'warning', 'error', 'fatal', 'none'", 0) \ + M(String, send_logs_source_regexp, "", "Send server text logs with specified regexp to match log source name. Empty means all sources.", 0) \ M(Bool, enable_optimize_predicate_expression, true, "If it is set to true, optimize predicates to subqueries.", 0) \ M(Bool, enable_optimize_predicate_expression_to_final_subquery, true, "Allow push predicate to final subquery.", 0) \ M(Bool, allow_push_predicate_when_subquery_contains_with, true, "Allows push predicate when subquery contains WITH clause", 0) \ diff --git a/src/Interpreters/InternalTextLogsQueue.cpp b/src/Interpreters/InternalTextLogsQueue.cpp index 2172a6f4261..8e689069cec 100644 --- a/src/Interpreters/InternalTextLogsQueue.cpp +++ b/src/Interpreters/InternalTextLogsQueue.cpp @@ -66,4 +66,19 @@ const char * InternalTextLogsQueue::getPriorityName(int priority) return (priority >= 1 && priority <= 8) ? PRIORITIES[priority] : PRIORITIES[0]; } +bool InternalTextLogsQueue::isNeeded(int priority, const String & source) const +{ + bool is_needed = priority <= max_priority; + + if (is_needed && source_regexp) + is_needed = re2::RE2::PartialMatch(source, *source_regexp); + + return is_needed; +} + +void InternalTextLogsQueue::setSourceRegexp(const String & regexp) +{ + source_regexp = std::make_unique(regexp); +} + } diff --git a/src/Interpreters/InternalTextLogsQueue.h b/src/Interpreters/InternalTextLogsQueue.h index 53710fa3bd2..9c3c9f9e707 100644 --- a/src/Interpreters/InternalTextLogsQueue.h +++ b/src/Interpreters/InternalTextLogsQueue.h @@ -2,7 +2,7 @@ #include #include #include - +#include namespace DB { @@ -15,6 +15,8 @@ public: InternalTextLogsQueue(); + bool isNeeded(int priority, const String & source) const; + static Block getSampleBlock(); static MutableColumns getSampleColumns(); @@ -23,6 +25,11 @@ public: /// Converts priority from Poco::Message::Priority to a string static const char * getPriorityName(int priority); + + void setSourceRegexp(const String & regexp); +private: + /// If not null, you should only push logs which are matched with this regexp + std::unique_ptr source_regexp; }; using InternalTextLogsQueuePtr = std::shared_ptr; diff --git a/src/Loggers/OwnSplitChannel.cpp b/src/Loggers/OwnSplitChannel.cpp index 71be8007d85..355b733b624 100644 --- a/src/Loggers/OwnSplitChannel.cpp +++ b/src/Loggers/OwnSplitChannel.cpp @@ -24,7 +24,7 @@ void OwnSplitChannel::log(const Poco::Message & msg) #ifdef WITH_TEXT_LOG auto logs_queue = CurrentThread::getInternalTextLogsQueue(); - if (channels.empty() && (logs_queue == nullptr || msg.getPriority() > logs_queue->max_priority)) + if (channels.empty() && (logs_queue == nullptr || !logs_queue->isNeeded(msg.getPriority(), msg.getSource()))) return; #endif @@ -93,7 +93,7 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg) auto logs_queue = CurrentThread::getInternalTextLogsQueue(); /// Log to "TCP queue" if message is not too noisy - if (logs_queue && msg.getPriority() <= logs_queue->max_priority) + if (logs_queue && logs_queue->isNeeded(msg.getPriority(), msg.getSource())) { MutableColumns columns = InternalTextLogsQueue::getSampleColumns(); diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index e1c73b7ebbb..4178d0d62da 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -848,6 +848,7 @@ namespace { logs_queue = std::make_shared(); logs_queue->max_priority = Poco::Logger::parseLevel(client_logs_level.toString()); + logs_queue->setSourceRegexp(settings.send_logs_source_regexp); CurrentThread::attachInternalTextLogsQueue(logs_queue, client_logs_level); CurrentThread::setFatalErrorCallback([this]{ onFatalError(); }); } diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index d1d762d3c61..05565063893 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -241,6 +241,7 @@ void TCPHandler::runImpl() { state.logs_queue = std::make_shared(); state.logs_queue->max_priority = Poco::Logger::parseLevel(client_logs_level.toString()); + state.logs_queue->setSourceRegexp(query_context->getSettingsRef().send_logs_source_regexp); CurrentThread::attachInternalTextLogsQueue(state.logs_queue, client_logs_level); CurrentThread::setFatalErrorCallback([this] { diff --git a/tests/queries/0_stateless/02359_send_logs_source_regexp.reference b/tests/queries/0_stateless/02359_send_logs_source_regexp.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02359_send_logs_source_regexp.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02359_send_logs_source_regexp.sh b/tests/queries/0_stateless/02359_send_logs_source_regexp.sh new file mode 100755 index 00000000000..d3b60bc59f4 --- /dev/null +++ b/tests/queries/0_stateless/02359_send_logs_source_regexp.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +[ ! -z "$CLICKHOUSE_CLIENT_REDEFINED" ] && CLICKHOUSE_CLIENT=$CLICKHOUSE_CLIENT_REDEFINED + +CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=trace/g') +regexp="executeQuery|InterpreterSelectQuery" +$CLICKHOUSE_CLIENT --send_logs_source_regexp "$regexp" -q "SELECT 1;" 2> >(grep -v -E "$regexp" 1>&2)