thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Structured-logging-custom-keys PR init

Browse Source
This commit is contained in:
root 2022-08-15 21:52:08 -07:00 committed by Mallik Hassan
parent d4a1b71b18
commit d5db88fa27
6 changed files with 213 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
programs/server/config.xml
View File

@ -65,9 +65,31 @@
in specified format like JSON.
For example, as below:
{"date_time":"1650918987.180175","thread_name":"#1","thread_id":"254545","level":"Trace","query_id":"","logger_name":"BaseDaemon","message":"Received signal 2","source_file":"../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()","source_line":"192"}
To enable JSON logging support, just uncomment <formatting> tag below.
To enable JSON logging support, please uncomment the entire <formatting> tag below.
a) You can modify key names by changing values under tag values inside <names> tag.
For example, to change DATE_TIME to MY_DATE_TIME, you can do like:
<date_time>MY_DATE_TIME</date_time>
b) You can stop unwanted log properties to appear in logs. To do so, you can simply comment out (recommended)
that property from this file.
For example, if you do not want your log to print query_id, you can comment out only <query_id> tag.
However, if you comment out all the tags under <names>, the program will print default values for as
below.
-->
<!-- <formatting>json</formatting> -->
<!-- <formatting>
<type>json</type>
<names>
<date_time>date_time</date_time>
<thread_name>thread_name</thread_name>
<thread_id>thread_id</thread_id>
<level>level</level>
<query_id>query_id</query_id>
<logger_name>logger_name</logger_name>
<message>message</message>
<source_file>source_file</source_file>
<source_line>source_line</source_line>
</names>
</formatting> -->
</logger>
<!-- Add headers to response in options request. OPTIONS method is used in CORS preflight requests. -->

4
src/Daemon/BaseDaemon.cpp
View File

@ -1016,8 +1016,8 @@ void BaseDaemon::setupWatchdog()
if (config().getRawString("logger.stream_compress", "false") == "true")
{
Poco::AutoPtr<OwnPatternFormatter> pf;
if (config().getString("logger.formatting", "") == "json")
pf = new OwnJSONPatternFormatter;
if (config().getString("logger.formatting.type", "") == "json")
pf = new OwnJSONPatternFormatter(config());
else
pf = new OwnPatternFormatter;
Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, new Poco::ConsoleChannel(std::cerr));

16
src/Loggers/Loggers.cpp
View File

@ -99,8 +99,8 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
Poco::AutoPtr<OwnPatternFormatter> pf;
if (config.getString("logger.formatting", "") == "json")
pf = new OwnJSONPatternFormatter;
if (config.getString("logger.formatting.type", "") == "json")
pf = new OwnJSONPatternFormatter(config);
else
pf = new OwnPatternFormatter;
@ -140,8 +140,8 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
Poco::AutoPtr<OwnPatternFormatter> pf;
if (config.getString("logger.formatting", "") == "json")
pf = new OwnJSONPatternFormatter;
if (config.getString("logger.formatting.type", "") == "json")
pf = new OwnJSONPatternFormatter(config);
else
pf = new OwnPatternFormatter;
@ -184,8 +184,8 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
Poco::AutoPtr<OwnPatternFormatter> pf;
if (config.getString("logger.formatting", "") == "json")
pf = new OwnJSONPatternFormatter;
if (config.getString("logger.formatting.type", "") == "json")
pf = new OwnJSONPatternFormatter(config);
else
pf = new OwnPatternFormatter;
@ -211,8 +211,8 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
}
Poco::AutoPtr<OwnPatternFormatter> pf;
if (config.getString("logger.formatting", "") == "json")
pf = new OwnJSONPatternFormatter;
if (config.getString("logger.formatting.type", "") == "json")
pf = new OwnJSONPatternFormatter(config);
else
pf = new OwnPatternFormatter(color_enabled);
Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, new Poco::ConsoleChannel);

209
src/Loggers/OwnJSONPatternFormatter.cpp
View File

@ -8,93 +8,186 @@
#include <Common/CurrentThread.h>
#include <Common/HashTable/Hash.h>
OwnJSONPatternFormatter::OwnJSONPatternFormatter() : OwnPatternFormatter("")
OwnJSONPatternFormatter::OwnJSONPatternFormatter(Poco::Util::AbstractConfiguration & config_) : OwnPatternFormatter(), config(config_)
{
}
if (config.has("logger.formatting.names.date_time"))
this->date_time = config.getString("logger.formatting.names.date_time", "");
if (config.has("logger.formatting.names.thread_name"))
this->thread_name = config.getString("logger.formatting.names.thread_name", "");
if (config.has("logger.formatting.names.thread_id"))
this->thread_id = config.getString("logger.formatting.names.thread_id", "");
if (config.has("logger.formatting.names.level"))
this->level = config.getString("logger.formatting.names.level", "");
if (config.has("logger.formatting.names.query_id"))
this->query_id = config.getString("logger.formatting.names.query_id", "");
if (config.has("logger.formatting.names.logger_name"))
this->logger_name = config.getString("logger.formatting.names.logger_name", "");
if (config.has("logger.formatting.names.message"))
this->message = config.getString("logger.formatting.names.message", "");
if (config.has("logger.formatting.names.source_file"))
this->source_file_ = config.getString("logger.formatting.names.source_file", "");
if (config.has("logger.formatting.names.source_line"))
this->source_line = config.getString("logger.formatting.names.source_line", "");
if (this->date_time.empty() && this->thread_name.empty() && this->thread_id.empty() && this->level.empty() && this->query_id.empty()
&& this->logger_name.empty() && this->message.empty() && this->source_file_.empty() && this->source_line.empty())
{
date_time = "date_time";
thread_name = "thread_name";
thread_id = "thread_id";
level = "level";
query_id = "query_id";
logger_name = "logger_name";
message = "message";
source_file_ = "source_file";
source_line = "source_line";
}
}
void OwnJSONPatternFormatter::formatExtended(const DB::ExtendedLogMessage & msg_ext, std::string & text) const
{
DB::WriteBufferFromString wb(text);
DB::FormatSettings settings;
bool print_comma = false;
const Poco::Message & msg = msg_ext.base;
DB::writeChar('{', wb);
writeJSONString("date_time", wb, settings);
DB::writeChar(':', wb);
if (!this->date_time.empty())
{
writeJSONString(this->date_time, wb, settings);
DB::writeChar(':', wb);
DB::writeChar('\"', wb);
/// Change delimiters in date for compatibility with old logs.
writeDateTimeUnixTimestamp(msg_ext.time_seconds, 0, wb);
DB::writeChar('.', wb);
DB::writeChar('0' + ((msg_ext.time_microseconds / 100000) % 10), wb);
DB::writeChar('0' + ((msg_ext.time_microseconds / 10000) % 10), wb);
DB::writeChar('0' + ((msg_ext.time_microseconds / 1000) % 10), wb);
DB::writeChar('0' + ((msg_ext.time_microseconds / 100) % 10), wb);
DB::writeChar('0' + ((msg_ext.time_microseconds / 10) % 10), wb);
DB::writeChar('0' + ((msg_ext.time_microseconds / 1) % 10), wb);
DB::writeChar('\"', wb);
DB::writeChar('\"', wb);
/// Change delimiters in date for compatibility with old logs.
writeDateTimeUnixTimestamp(msg_ext.time_seconds, 0, wb);
DB::writeChar('.', wb);
DB::writeChar('0' + ((msg_ext.time_microseconds / 100000) % 10), wb);
DB::writeChar('0' + ((msg_ext.time_microseconds / 10000) % 10), wb);
DB::writeChar('0' + ((msg_ext.time_microseconds / 1000) % 10), wb);
DB::writeChar('0' + ((msg_ext.time_microseconds / 100) % 10), wb);
DB::writeChar('0' + ((msg_ext.time_microseconds / 10) % 10), wb);
DB::writeChar('0' + ((msg_ext.time_microseconds / 1) % 10), wb);
DB::writeChar('\"', wb);
print_comma = true;
}
DB::writeChar(',', wb);
if (!this->thread_name.empty())
{
if (print_comma)
DB::writeChar(',', wb);
else
print_comma = true;
writeJSONString("thread_name", wb, settings);
DB::writeChar(':', wb);
writeJSONString(this->thread_name, wb, settings);
DB::writeChar(':', wb);
writeJSONString(msg.getThread(), wb, settings);
writeJSONString(msg.getThread(), wb, settings);
}
DB::writeChar(',', wb);
if (!this->thread_id.empty())
{
if (print_comma)
DB::writeChar(',', wb);
else
print_comma = true;
writeJSONString("thread_id", wb, settings);
DB::writeChar(':', wb);
DB::writeChar('\"', wb);
DB::writeIntText(msg_ext.thread_id, wb);
DB::writeChar('\"', wb);
writeJSONString(this->thread_id, wb, settings);
DB::writeChar(':', wb);
DB::writeChar('\"', wb);
DB::writeIntText(msg_ext.thread_id, wb);
DB::writeChar('\"', wb);
}
DB::writeChar(',', wb);
if (!this->level.empty())
{
if (print_comma)
DB::writeChar(',', wb);
else
print_comma = true;
writeJSONString("level", wb, settings);
DB::writeChar(':', wb);
int priority = static_cast<int>(msg.getPriority());
writeJSONString(std::to_string(priority), wb, settings);
DB::writeChar(',', wb);
writeJSONString(this->level, wb, settings);
DB::writeChar(':', wb);
int priority = static_cast<int>(msg.getPriority());
writeJSONString(std::to_string(priority), wb, settings);
}
/// We write query_id even in case when it is empty (no query context)
/// just to be convenient for various log parsers.
if (!this->query_id.empty())
{
if (print_comma)
DB::writeChar(',', wb);
else
print_comma = true;
writeJSONString("query_id", wb, settings);
DB::writeChar(':', wb);
writeJSONString(msg_ext.query_id, wb, settings);
/// We write query_id even in case when it is empty (no query context)
/// just to be convenient for various log parsers.
DB::writeChar(',', wb);
writeJSONString(this->query_id, wb, settings);
DB::writeChar(':', wb);
writeJSONString(msg_ext.query_id, wb, settings);
}
writeJSONString("logger_name", wb, settings);
DB::writeChar(':', wb);
if (!this->logger_name.empty())
{
if (print_comma)
DB::writeChar(',', wb);
else
print_comma = true;
writeJSONString(msg.getSource(), wb, settings);
DB::writeChar(',', wb);
writeJSONString(this->logger_name, wb, settings);
DB::writeChar(':', wb);
writeJSONString("message", wb, settings);
DB::writeChar(':', wb);
writeJSONString(msg.getText(), wb, settings);
DB::writeChar(',', wb);
writeJSONString(msg.getSource(), wb, settings);
}
writeJSONString("source_file", wb, settings);
DB::writeChar(':', wb);
const char * source_file = msg.getSourceFile();
if (source_file != nullptr)
writeJSONString(source_file, wb, settings);
else
writeJSONString("", wb, settings);
DB::writeChar(',', wb);
if (!this->message.empty())
{
if (print_comma)
DB::writeChar(',', wb);
else
print_comma = true;
writeJSONString("source_line", wb, settings);
DB::writeChar(':', wb);
DB::writeChar('\"', wb);
DB::writeIntText(msg.getSourceLine(), wb);
DB::writeChar('\"', wb);
writeJSONString(this->message, wb, settings);
DB::writeChar(':', wb);
writeJSONString(msg.getText(), wb, settings);
}
if (!this->source_file_.empty())
{
if (print_comma)
DB::writeChar(',', wb);
else
print_comma = true;
writeJSONString(this->source_file_, wb, settings);
DB::writeChar(':', wb);
const char * source_file = msg.getSourceFile();
if (source_file != nullptr)
writeJSONString(source_file, wb, settings);
else
writeJSONString("", wb, settings);
}
if (!this->source_line.empty())
{
if (print_comma)
DB::writeChar(',', wb);
writeJSONString(this->source_line, wb, settings);
DB::writeChar(':', wb);
DB::writeChar('\"', wb);
DB::writeIntText(msg.getSourceLine(), wb);
DB::writeChar('\"', wb);
}
DB::writeChar('}', wb);
}

16
src/Loggers/OwnJSONPatternFormatter.h
View File

@ -24,9 +24,21 @@ class Loggers;
class OwnJSONPatternFormatter : public OwnPatternFormatter
{
public:
OwnJSONPatternFormatter();
public:
OwnJSONPatternFormatter(Poco::Util::AbstractConfiguration & config_);
void format(const Poco::Message & msg, std::string & text) override;
void formatExtended(const DB::ExtendedLogMessage & msg_ext, std::string & text) const override;
private:
Poco::Util::AbstractConfiguration & config;
std::string date_time;
std::string thread_name;
std::string thread_id;
std::string level;
std::string query_id;
std::string logger_name;
std::string message;
std::string source_file_;
std::string source_line;
};

15
tests/integration/test_structured_logging_json/configs/config_json.xml
View File

@ -8,7 +8,20 @@
{"date_time":"1650918987.180175","thread_name":"#1","thread_id":"254545","level":"Trace","query_id":"","logger_name":"BaseDaemon","message":"Received signal 2","source_file":"../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()","source_line":"192"}
To enable JSON logging support, just uncomment <formatting> tag below.
-->
<formatting>json</formatting>
<formatting>
<type>json</type>
<names>
<date_time>DATE_TIME</date_time>
<thread_name>THREAD_NAME</thread_name>
<thread_id>THREAD_ID</thread_id>
<level>LEVEL</level>
<query_id>QUERY_ID</query_id>
<logger_name>LOGGER_NAME</logger_name>
<message>MESSAGE</message>
<source_file>SOURCE_FILE</source_file>
<source_line>SOURCE_LINE</source_line>
</names>
</formatting>
</logger>
</clickhouse>