diff --git a/programs/server/config.xml b/programs/server/config.xml index a29c619b1dd..07d1b70d15a 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -65,9 +65,31 @@ in specified format like JSON. For example, as below: {"date_time":"1650918987.180175","thread_name":"#1","thread_id":"254545","level":"Trace","query_id":"","logger_name":"BaseDaemon","message":"Received signal 2","source_file":"../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()","source_line":"192"} - To enable JSON logging support, just uncomment tag below. + To enable JSON logging support, please uncomment the entire tag below. + + a) You can modify key names by changing values under tag values inside tag. + For example, to change DATE_TIME to MY_DATE_TIME, you can do like: + MY_DATE_TIME + b) You can stop unwanted log properties to appear in logs. To do so, you can simply comment out (recommended) + that property from this file. + For example, if you do not want your log to print query_id, you can comment out only tag. + However, if you comment out all the tags under , the program will print default values for as + below. --> - + diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index 19c61e434ae..d449768935a 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -1016,8 +1016,8 @@ void BaseDaemon::setupWatchdog() if (config().getRawString("logger.stream_compress", "false") == "true") { Poco::AutoPtr pf; - if (config().getString("logger.formatting", "") == "json") - pf = new OwnJSONPatternFormatter; + if (config().getString("logger.formatting.type", "") == "json") + pf = new OwnJSONPatternFormatter(config()); else pf = new OwnPatternFormatter; Poco::AutoPtr log = new DB::OwnFormattingChannel(pf, new Poco::ConsoleChannel(std::cerr)); diff --git a/src/Loggers/Loggers.cpp b/src/Loggers/Loggers.cpp index dec6bcd51c7..747496c5754 100644 --- a/src/Loggers/Loggers.cpp +++ b/src/Loggers/Loggers.cpp @@ -99,8 +99,8 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log Poco::AutoPtr pf; - if (config.getString("logger.formatting", "") == "json") - pf = new OwnJSONPatternFormatter; + if (config.getString("logger.formatting.type", "") == "json") + pf = new OwnJSONPatternFormatter(config); else pf = new OwnPatternFormatter; @@ -140,8 +140,8 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log Poco::AutoPtr pf; - if (config.getString("logger.formatting", "") == "json") - pf = new OwnJSONPatternFormatter; + if (config.getString("logger.formatting.type", "") == "json") + pf = new OwnJSONPatternFormatter(config); else pf = new OwnPatternFormatter; @@ -184,8 +184,8 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log Poco::AutoPtr pf; - if (config.getString("logger.formatting", "") == "json") - pf = new OwnJSONPatternFormatter; + if (config.getString("logger.formatting.type", "") == "json") + pf = new OwnJSONPatternFormatter(config); else pf = new OwnPatternFormatter; @@ -211,8 +211,8 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log } Poco::AutoPtr pf; - if (config.getString("logger.formatting", "") == "json") - pf = new OwnJSONPatternFormatter; + if (config.getString("logger.formatting.type", "") == "json") + pf = new OwnJSONPatternFormatter(config); else pf = new OwnPatternFormatter(color_enabled); Poco::AutoPtr log = new DB::OwnFormattingChannel(pf, new Poco::ConsoleChannel); diff --git a/src/Loggers/OwnJSONPatternFormatter.cpp b/src/Loggers/OwnJSONPatternFormatter.cpp index d12506e2451..143b75762b1 100644 --- a/src/Loggers/OwnJSONPatternFormatter.cpp +++ b/src/Loggers/OwnJSONPatternFormatter.cpp @@ -8,93 +8,186 @@ #include #include -OwnJSONPatternFormatter::OwnJSONPatternFormatter() : OwnPatternFormatter("") +OwnJSONPatternFormatter::OwnJSONPatternFormatter(Poco::Util::AbstractConfiguration & config_) : OwnPatternFormatter(), config(config_) { -} + if (config.has("logger.formatting.names.date_time")) + this->date_time = config.getString("logger.formatting.names.date_time", ""); + if (config.has("logger.formatting.names.thread_name")) + this->thread_name = config.getString("logger.formatting.names.thread_name", ""); + + if (config.has("logger.formatting.names.thread_id")) + this->thread_id = config.getString("logger.formatting.names.thread_id", ""); + + if (config.has("logger.formatting.names.level")) + this->level = config.getString("logger.formatting.names.level", ""); + + if (config.has("logger.formatting.names.query_id")) + this->query_id = config.getString("logger.formatting.names.query_id", ""); + + if (config.has("logger.formatting.names.logger_name")) + this->logger_name = config.getString("logger.formatting.names.logger_name", ""); + + if (config.has("logger.formatting.names.message")) + this->message = config.getString("logger.formatting.names.message", ""); + + if (config.has("logger.formatting.names.source_file")) + this->source_file_ = config.getString("logger.formatting.names.source_file", ""); + + if (config.has("logger.formatting.names.source_line")) + this->source_line = config.getString("logger.formatting.names.source_line", ""); + + if (this->date_time.empty() && this->thread_name.empty() && this->thread_id.empty() && this->level.empty() && this->query_id.empty() + && this->logger_name.empty() && this->message.empty() && this->source_file_.empty() && this->source_line.empty()) + { + date_time = "date_time"; + thread_name = "thread_name"; + thread_id = "thread_id"; + level = "level"; + query_id = "query_id"; + logger_name = "logger_name"; + message = "message"; + source_file_ = "source_file"; + source_line = "source_line"; + } +} void OwnJSONPatternFormatter::formatExtended(const DB::ExtendedLogMessage & msg_ext, std::string & text) const { DB::WriteBufferFromString wb(text); DB::FormatSettings settings; + bool print_comma = false; const Poco::Message & msg = msg_ext.base; DB::writeChar('{', wb); - writeJSONString("date_time", wb, settings); - DB::writeChar(':', wb); + if (!this->date_time.empty()) + { + writeJSONString(this->date_time, wb, settings); + DB::writeChar(':', wb); - DB::writeChar('\"', wb); - /// Change delimiters in date for compatibility with old logs. - writeDateTimeUnixTimestamp(msg_ext.time_seconds, 0, wb); - DB::writeChar('.', wb); - DB::writeChar('0' + ((msg_ext.time_microseconds / 100000) % 10), wb); - DB::writeChar('0' + ((msg_ext.time_microseconds / 10000) % 10), wb); - DB::writeChar('0' + ((msg_ext.time_microseconds / 1000) % 10), wb); - DB::writeChar('0' + ((msg_ext.time_microseconds / 100) % 10), wb); - DB::writeChar('0' + ((msg_ext.time_microseconds / 10) % 10), wb); - DB::writeChar('0' + ((msg_ext.time_microseconds / 1) % 10), wb); - DB::writeChar('\"', wb); + DB::writeChar('\"', wb); + /// Change delimiters in date for compatibility with old logs. + writeDateTimeUnixTimestamp(msg_ext.time_seconds, 0, wb); + DB::writeChar('.', wb); + DB::writeChar('0' + ((msg_ext.time_microseconds / 100000) % 10), wb); + DB::writeChar('0' + ((msg_ext.time_microseconds / 10000) % 10), wb); + DB::writeChar('0' + ((msg_ext.time_microseconds / 1000) % 10), wb); + DB::writeChar('0' + ((msg_ext.time_microseconds / 100) % 10), wb); + DB::writeChar('0' + ((msg_ext.time_microseconds / 10) % 10), wb); + DB::writeChar('0' + ((msg_ext.time_microseconds / 1) % 10), wb); + DB::writeChar('\"', wb); + print_comma = true; + } - DB::writeChar(',', wb); + if (!this->thread_name.empty()) + { + if (print_comma) + DB::writeChar(',', wb); + else + print_comma = true; - writeJSONString("thread_name", wb, settings); - DB::writeChar(':', wb); + writeJSONString(this->thread_name, wb, settings); + DB::writeChar(':', wb); - writeJSONString(msg.getThread(), wb, settings); + writeJSONString(msg.getThread(), wb, settings); + } - DB::writeChar(',', wb); + if (!this->thread_id.empty()) + { + if (print_comma) + DB::writeChar(',', wb); + else + print_comma = true; - writeJSONString("thread_id", wb, settings); - DB::writeChar(':', wb); - DB::writeChar('\"', wb); - DB::writeIntText(msg_ext.thread_id, wb); - DB::writeChar('\"', wb); + writeJSONString(this->thread_id, wb, settings); + DB::writeChar(':', wb); + DB::writeChar('\"', wb); + DB::writeIntText(msg_ext.thread_id, wb); + DB::writeChar('\"', wb); + } - DB::writeChar(',', wb); + if (!this->level.empty()) + { + if (print_comma) + DB::writeChar(',', wb); + else + print_comma = true; - writeJSONString("level", wb, settings); - DB::writeChar(':', wb); - int priority = static_cast(msg.getPriority()); - writeJSONString(std::to_string(priority), wb, settings); - DB::writeChar(',', wb); + writeJSONString(this->level, wb, settings); + DB::writeChar(':', wb); + int priority = static_cast(msg.getPriority()); + writeJSONString(std::to_string(priority), wb, settings); + } - /// We write query_id even in case when it is empty (no query context) - /// just to be convenient for various log parsers. + if (!this->query_id.empty()) + { + if (print_comma) + DB::writeChar(',', wb); + else + print_comma = true; - writeJSONString("query_id", wb, settings); - DB::writeChar(':', wb); - writeJSONString(msg_ext.query_id, wb, settings); + /// We write query_id even in case when it is empty (no query context) + /// just to be convenient for various log parsers. - DB::writeChar(',', wb); + writeJSONString(this->query_id, wb, settings); + DB::writeChar(':', wb); + writeJSONString(msg_ext.query_id, wb, settings); + } - writeJSONString("logger_name", wb, settings); - DB::writeChar(':', wb); + if (!this->logger_name.empty()) + { + if (print_comma) + DB::writeChar(',', wb); + else + print_comma = true; - writeJSONString(msg.getSource(), wb, settings); - DB::writeChar(',', wb); + writeJSONString(this->logger_name, wb, settings); + DB::writeChar(':', wb); - writeJSONString("message", wb, settings); - DB::writeChar(':', wb); - writeJSONString(msg.getText(), wb, settings); - DB::writeChar(',', wb); + writeJSONString(msg.getSource(), wb, settings); + } - writeJSONString("source_file", wb, settings); - DB::writeChar(':', wb); - const char * source_file = msg.getSourceFile(); - if (source_file != nullptr) - writeJSONString(source_file, wb, settings); - else - writeJSONString("", wb, settings); - DB::writeChar(',', wb); + if (!this->message.empty()) + { + if (print_comma) + DB::writeChar(',', wb); + else + print_comma = true; - writeJSONString("source_line", wb, settings); - DB::writeChar(':', wb); - DB::writeChar('\"', wb); - DB::writeIntText(msg.getSourceLine(), wb); - DB::writeChar('\"', wb); + writeJSONString(this->message, wb, settings); + DB::writeChar(':', wb); + writeJSONString(msg.getText(), wb, settings); + } + if (!this->source_file_.empty()) + { + if (print_comma) + DB::writeChar(',', wb); + else + print_comma = true; + + writeJSONString(this->source_file_, wb, settings); + DB::writeChar(':', wb); + const char * source_file = msg.getSourceFile(); + if (source_file != nullptr) + writeJSONString(source_file, wb, settings); + else + writeJSONString("", wb, settings); + } + + if (!this->source_line.empty()) + { + if (print_comma) + DB::writeChar(',', wb); + + writeJSONString(this->source_line, wb, settings); + DB::writeChar(':', wb); + DB::writeChar('\"', wb); + DB::writeIntText(msg.getSourceLine(), wb); + DB::writeChar('\"', wb); + } DB::writeChar('}', wb); } diff --git a/src/Loggers/OwnJSONPatternFormatter.h b/src/Loggers/OwnJSONPatternFormatter.h index 032506b15e3..72930de976d 100644 --- a/src/Loggers/OwnJSONPatternFormatter.h +++ b/src/Loggers/OwnJSONPatternFormatter.h @@ -24,9 +24,21 @@ class Loggers; class OwnJSONPatternFormatter : public OwnPatternFormatter { -public: - OwnJSONPatternFormatter(); + public: + OwnJSONPatternFormatter(Poco::Util::AbstractConfiguration & config_); void format(const Poco::Message & msg, std::string & text) override; void formatExtended(const DB::ExtendedLogMessage & msg_ext, std::string & text) const override; + + private: + Poco::Util::AbstractConfiguration & config; + std::string date_time; + std::string thread_name; + std::string thread_id; + std::string level; + std::string query_id; + std::string logger_name; + std::string message; + std::string source_file_; + std::string source_line; }; diff --git a/tests/integration/test_structured_logging_json/configs/config_json.xml b/tests/integration/test_structured_logging_json/configs/config_json.xml index c0bd4e86a01..f20fda50319 100644 --- a/tests/integration/test_structured_logging_json/configs/config_json.xml +++ b/tests/integration/test_structured_logging_json/configs/config_json.xml @@ -8,7 +8,20 @@ {"date_time":"1650918987.180175","thread_name":"#1","thread_id":"254545","level":"Trace","query_id":"","logger_name":"BaseDaemon","message":"Received signal 2","source_file":"../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()","source_line":"192"} To enable JSON logging support, just uncomment tag below. --> - json + + json + + DATE_TIME + THREAD_NAME + THREAD_ID + LEVEL + QUERY_ID + LOGGER_NAME + MESSAGE + SOURCE_FILE + SOURCE_LINE + +