Structured-logging-custom-keys PR init

programs/server/config.xml

@@ -65,9 +65,31 @@
         in specified format like JSON.
         For example, as below:
         {"date_time":"1650918987.180175","thread_name":"#1","thread_id":"254545","level":"Trace","query_id":"","logger_name":"BaseDaemon","message":"Received signal 2","source_file":"../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()","source_line":"192"}
-        To enable JSON logging support, just uncomment <formatting> tag below.
+        To enable JSON logging support, please uncomment the entire <formatting> tag below.
+        
+        a) You can modify key names by changing values under tag values inside <names> tag.
+        For example, to change DATE_TIME to MY_DATE_TIME, you can do like:
+            <date_time>MY_DATE_TIME</date_time>
+        b) You can stop unwanted log properties to appear in logs. To do so, you can simply comment out (recommended)
+        that property from this file.
+        For example, if you do not want your log to print query_id, you can comment out only <query_id> tag.
+        However, if you comment out all the tags under <names>, the program will print default values for as
+        below.
         -->
-        <!-- <formatting>json</formatting> -->
+        <!-- <formatting>
+            <type>json</type>
+            <names>
+                <date_time>date_time</date_time>
+                <thread_name>thread_name</thread_name>
+                <thread_id>thread_id</thread_id>
+                <level>level</level>
+                <query_id>query_id</query_id>
+                <logger_name>logger_name</logger_name>
+                <message>message</message>
+                <source_file>source_file</source_file>
+                <source_line>source_line</source_line>
+            </names>
+        </formatting> -->
     </logger>
 
     <!-- Add headers to response in options request. OPTIONS method is used in CORS preflight requests. -->

src/Daemon/BaseDaemon.cpp

@@ -1016,8 +1016,8 @@ void BaseDaemon::setupWatchdog()
         if (config().getRawString("logger.stream_compress", "false") == "true")
         {
             Poco::AutoPtr<OwnPatternFormatter> pf;
-            if (config().getString("logger.formatting", "") == "json")
+            if (config().getString("logger.formatting.type", "") == "json")
-                pf = new OwnJSONPatternFormatter;
+                pf = new OwnJSONPatternFormatter(config());
             else
                 pf = new OwnPatternFormatter;
             Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, new Poco::ConsoleChannel(std::cerr));

src/Loggers/Loggers.cpp

@@ -99,8 +99,8 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
 
         Poco::AutoPtr<OwnPatternFormatter> pf;
 
-        if (config.getString("logger.formatting", "") == "json")
+        if (config.getString("logger.formatting.type", "") == "json")
-            pf = new OwnJSONPatternFormatter;
+           pf = new OwnJSONPatternFormatter(config);
         else
             pf = new OwnPatternFormatter;
 
@@ -140,8 +140,8 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
 
         Poco::AutoPtr<OwnPatternFormatter> pf;
 
-        if (config.getString("logger.formatting", "") == "json")
+        if (config.getString("logger.formatting.type", "") == "json")
-            pf = new OwnJSONPatternFormatter;
+            pf = new OwnJSONPatternFormatter(config);
         else
             pf = new OwnPatternFormatter;
 
@@ -184,8 +184,8 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
 
         Poco::AutoPtr<OwnPatternFormatter> pf;
 
-        if (config.getString("logger.formatting", "") == "json")
+        if (config.getString("logger.formatting.type", "") == "json")
-            pf = new OwnJSONPatternFormatter;
+            pf = new OwnJSONPatternFormatter(config);
         else
             pf = new OwnPatternFormatter;
 
@@ -211,8 +211,8 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
         }
 
         Poco::AutoPtr<OwnPatternFormatter> pf;
-        if (config.getString("logger.formatting", "") == "json")
+        if (config.getString("logger.formatting.type", "") == "json")
-            pf = new OwnJSONPatternFormatter;
+            pf = new OwnJSONPatternFormatter(config);
         else
             pf = new OwnPatternFormatter(color_enabled);
         Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, new Poco::ConsoleChannel);

src/Loggers/OwnJSONPatternFormatter.cpp

@@ -8,93 +8,186 @@
 #include <Common/CurrentThread.h>
 #include <Common/HashTable/Hash.h>
 
-OwnJSONPatternFormatter::OwnJSONPatternFormatter() : OwnPatternFormatter("")
+OwnJSONPatternFormatter::OwnJSONPatternFormatter(Poco::Util::AbstractConfiguration & config_) : OwnPatternFormatter(), config(config_)
 {
-}
+    if (config.has("logger.formatting.names.date_time"))
+        this->date_time = config.getString("logger.formatting.names.date_time", "");
 
+    if (config.has("logger.formatting.names.thread_name"))
+        this->thread_name = config.getString("logger.formatting.names.thread_name", "");
+
+    if (config.has("logger.formatting.names.thread_id"))
+        this->thread_id = config.getString("logger.formatting.names.thread_id", "");
+
+    if (config.has("logger.formatting.names.level"))
+        this->level = config.getString("logger.formatting.names.level", "");
+
+    if (config.has("logger.formatting.names.query_id"))
+        this->query_id = config.getString("logger.formatting.names.query_id", "");
+
+    if (config.has("logger.formatting.names.logger_name"))
+        this->logger_name = config.getString("logger.formatting.names.logger_name", "");
+
+    if (config.has("logger.formatting.names.message"))
+        this->message = config.getString("logger.formatting.names.message", "");
+
+    if (config.has("logger.formatting.names.source_file"))
+        this->source_file_ = config.getString("logger.formatting.names.source_file", "");
+
+    if (config.has("logger.formatting.names.source_line"))
+        this->source_line = config.getString("logger.formatting.names.source_line", "");
+
+    if (this->date_time.empty() && this->thread_name.empty() && this->thread_id.empty() && this->level.empty() && this->query_id.empty()
+        && this->logger_name.empty() && this->message.empty() && this->source_file_.empty() && this->source_line.empty())
+    {
+        date_time = "date_time";
+        thread_name = "thread_name";
+        thread_id = "thread_id";
+        level = "level";
+        query_id = "query_id";
+        logger_name = "logger_name";
+        message = "message";
+        source_file_ = "source_file";
+        source_line = "source_line";
+    }
+}
 
 void OwnJSONPatternFormatter::formatExtended(const DB::ExtendedLogMessage & msg_ext, std::string & text) const
 {
     DB::WriteBufferFromString wb(text);
 
     DB::FormatSettings settings;
+    bool print_comma = false;
 
     const Poco::Message & msg = msg_ext.base;
     DB::writeChar('{', wb);
 
-    writeJSONString("date_time", wb, settings);
+    if (!this->date_time.empty())
-    DB::writeChar(':', wb);
+    {
+        writeJSONString(this->date_time, wb, settings);
+        DB::writeChar(':', wb);
 
-    DB::writeChar('\"', wb);
+        DB::writeChar('\"', wb);
-    /// Change delimiters in date for compatibility with old logs.
+        /// Change delimiters in date for compatibility with old logs.
-    writeDateTimeUnixTimestamp(msg_ext.time_seconds, 0, wb);
+        writeDateTimeUnixTimestamp(msg_ext.time_seconds, 0, wb);
-    DB::writeChar('.', wb);
+        DB::writeChar('.', wb);
-    DB::writeChar('0' + ((msg_ext.time_microseconds / 100000) % 10), wb);
+        DB::writeChar('0' + ((msg_ext.time_microseconds / 100000) % 10), wb);
-    DB::writeChar('0' + ((msg_ext.time_microseconds / 10000) % 10), wb);
+        DB::writeChar('0' + ((msg_ext.time_microseconds / 10000) % 10), wb);
-    DB::writeChar('0' + ((msg_ext.time_microseconds / 1000) % 10), wb);
+        DB::writeChar('0' + ((msg_ext.time_microseconds / 1000) % 10), wb);
-    DB::writeChar('0' + ((msg_ext.time_microseconds / 100) % 10), wb);
+        DB::writeChar('0' + ((msg_ext.time_microseconds / 100) % 10), wb);
-    DB::writeChar('0' + ((msg_ext.time_microseconds / 10) % 10), wb);
+        DB::writeChar('0' + ((msg_ext.time_microseconds / 10) % 10), wb);
-    DB::writeChar('0' + ((msg_ext.time_microseconds / 1) % 10), wb);
+        DB::writeChar('0' + ((msg_ext.time_microseconds / 1) % 10), wb);
-    DB::writeChar('\"', wb);
+        DB::writeChar('\"', wb);
+        print_comma = true;
+    }
 
-    DB::writeChar(',', wb);
+    if (!this->thread_name.empty())
+    {
+        if (print_comma)
+            DB::writeChar(',', wb);
+        else
+            print_comma = true;
 
-    writeJSONString("thread_name", wb, settings);
+        writeJSONString(this->thread_name, wb, settings);
-    DB::writeChar(':', wb);
+        DB::writeChar(':', wb);
 
-    writeJSONString(msg.getThread(), wb, settings);
+        writeJSONString(msg.getThread(), wb, settings);
+    }
 
-    DB::writeChar(',', wb);
+    if (!this->thread_id.empty())
+    {
+        if (print_comma)
+            DB::writeChar(',', wb);
+        else
+            print_comma = true;
 
-    writeJSONString("thread_id", wb, settings);
+        writeJSONString(this->thread_id, wb, settings);
-    DB::writeChar(':', wb);
+        DB::writeChar(':', wb);
-    DB::writeChar('\"', wb);
+        DB::writeChar('\"', wb);
-    DB::writeIntText(msg_ext.thread_id, wb);
+        DB::writeIntText(msg_ext.thread_id, wb);
-    DB::writeChar('\"', wb);
+        DB::writeChar('\"', wb);
+    }
 
-    DB::writeChar(',', wb);
+    if (!this->level.empty())
+    {
+        if (print_comma)
+            DB::writeChar(',', wb);
+        else
+            print_comma = true;
 
-    writeJSONString("level", wb, settings);
+        writeJSONString(this->level, wb, settings);
-    DB::writeChar(':', wb);
+        DB::writeChar(':', wb);
-    int priority = static_cast<int>(msg.getPriority());
+        int priority = static_cast<int>(msg.getPriority());
-    writeJSONString(std::to_string(priority), wb, settings);
+        writeJSONString(std::to_string(priority), wb, settings);
-    DB::writeChar(',', wb);
+    }
 
-    /// We write query_id even in case when it is empty (no query context)
+    if (!this->query_id.empty())
-    /// just to be convenient for various log parsers.
+    {
+        if (print_comma)
+            DB::writeChar(',', wb);
+        else
+            print_comma = true;
 
-    writeJSONString("query_id", wb, settings);
+        /// We write query_id even in case when it is empty (no query context)
-    DB::writeChar(':', wb);
+        /// just to be convenient for various log parsers.
-    writeJSONString(msg_ext.query_id, wb, settings);
 
-    DB::writeChar(',', wb);
+        writeJSONString(this->query_id, wb, settings);
+        DB::writeChar(':', wb);
+        writeJSONString(msg_ext.query_id, wb, settings);
+    }
 
-    writeJSONString("logger_name", wb, settings);
+    if (!this->logger_name.empty())
-    DB::writeChar(':', wb);
+    {
+        if (print_comma)
+            DB::writeChar(',', wb);
+        else
+            print_comma = true;
 
-    writeJSONString(msg.getSource(), wb, settings);
+        writeJSONString(this->logger_name, wb, settings);
-    DB::writeChar(',', wb);
+        DB::writeChar(':', wb);
 
-    writeJSONString("message", wb, settings);
+        writeJSONString(msg.getSource(), wb, settings);
-    DB::writeChar(':', wb);
+    }
-    writeJSONString(msg.getText(), wb, settings);
-    DB::writeChar(',', wb);
 
-    writeJSONString("source_file", wb, settings);
+    if (!this->message.empty())
-    DB::writeChar(':', wb);
+    {
-    const char * source_file = msg.getSourceFile();
+        if (print_comma)
-    if (source_file != nullptr)
+            DB::writeChar(',', wb);
-        writeJSONString(source_file, wb, settings);
+        else
-    else
+            print_comma = true;
-        writeJSONString("", wb, settings);
-    DB::writeChar(',', wb);
 
-    writeJSONString("source_line", wb, settings);
+        writeJSONString(this->message, wb, settings);
-    DB::writeChar(':', wb);
+        DB::writeChar(':', wb);
-    DB::writeChar('\"', wb);
+        writeJSONString(msg.getText(), wb, settings);
-    DB::writeIntText(msg.getSourceLine(), wb);
+    }
-    DB::writeChar('\"', wb);
 
+    if (!this->source_file_.empty())
+    {
+        if (print_comma)
+            DB::writeChar(',', wb);
+        else
+            print_comma = true;
+
+        writeJSONString(this->source_file_, wb, settings);
+        DB::writeChar(':', wb);
+        const char * source_file = msg.getSourceFile();
+        if (source_file != nullptr)
+            writeJSONString(source_file, wb, settings);
+        else
+            writeJSONString("", wb, settings);
+    }
+
+    if (!this->source_line.empty())
+    {
+        if (print_comma)
+            DB::writeChar(',', wb);
+
+        writeJSONString(this->source_line, wb, settings);
+        DB::writeChar(':', wb);
+        DB::writeChar('\"', wb);
+        DB::writeIntText(msg.getSourceLine(), wb);
+        DB::writeChar('\"', wb);
+    }
     DB::writeChar('}', wb);
 }
 

src/Loggers/OwnJSONPatternFormatter.h

@@ -24,9 +24,21 @@ class Loggers;
 
 class OwnJSONPatternFormatter : public OwnPatternFormatter
 {
-public:
+  public:
-    OwnJSONPatternFormatter();
+    OwnJSONPatternFormatter(Poco::Util::AbstractConfiguration & config_);
 
     void format(const Poco::Message & msg, std::string & text) override;
     void formatExtended(const DB::ExtendedLogMessage & msg_ext, std::string & text) const override;
+  
+  private:
+    Poco::Util::AbstractConfiguration & config;
+    std::string date_time;
+    std::string thread_name;
+    std::string thread_id;
+    std::string level;
+    std::string query_id;
+    std::string logger_name;
+    std::string message;
+    std::string source_file_;
+    std::string source_line;
 };

tests/integration/test_structured_logging_json/configs/config_json.xml

@@ -8,7 +8,20 @@
         {"date_time":"1650918987.180175","thread_name":"#1","thread_id":"254545","level":"Trace","query_id":"","logger_name":"BaseDaemon","message":"Received signal 2","source_file":"../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()","source_line":"192"}
         To enable JSON logging support, just uncomment <formatting> tag below.
         -->
-        <formatting>json</formatting>
+        <formatting>
+            <type>json</type>
+            <names>
+                <date_time>DATE_TIME</date_time>
+                <thread_name>THREAD_NAME</thread_name>
+                <thread_id>THREAD_ID</thread_id>
+                <level>LEVEL</level>
+                <query_id>QUERY_ID</query_id>
+                <logger_name>LOGGER_NAME</logger_name>
+                <message>MESSAGE</message>
+                <source_file>SOURCE_FILE</source_file>
+                <source_line>SOURCE_LINE</source_line>
+            </names>
+        </formatting>
     </logger>
 
 </clickhouse>