mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 01:25:21 +00:00
Modified test code to cover various scenarios for custom keys
This commit is contained in:
root
Mallik Hassan
parent
834ab5cd2b
commit
1d29a494db
@ -0,0 +1,27 @@
|
||||
<?xml version="1.0"?>
|
||||
<clickhouse>
|
||||
<logger>
|
||||
<!-- Structured log formatting:
|
||||
You can specify log format(for now, JSON only). In that case, the console log will be printed
|
||||
in specified format like JSON.
|
||||
For example, as below:
|
||||
{"date_time":"1650918987.180175","thread_name":"#1","thread_id":"254545","level":"Trace","query_id":"","logger_name":"BaseDaemon","message":"Received signal 2","source_file":"../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()","source_line":"192"}
|
||||
To enable JSON logging support, just uncomment <formatting> tag below.
|
||||
-->
|
||||
<formatting>
|
||||
<type>json</type>
|
||||
<names>
|
||||
<date_time>DATE_TIME</date_time>
|
||||
<thread_name>THREAD_NAME</thread_name>
|
||||
<thread_id>THREAD_ID</thread_id>
|
||||
<level>LEVEL</level>
|
||||
<query_id>QUERY_ID</query_id>
|
||||
<logger_name>LOGGER_NAME</logger_name>
|
||||
<message>MESSAGE</message>
|
||||
<source_file>SOURCE_FILE</source_file>
|
||||
<source_line>SOURCE_LINE</source_line>
|
||||
</names>
|
||||
</formatting>
|
||||
</logger>
|
||||
|
||||
</clickhouse>
|
||||
@ -0,0 +1,27 @@
|
||||
<?xml version="1.0"?>
|
||||
<clickhouse>
|
||||
<logger>
|
||||
<!-- Structured log formatting:
|
||||
You can specify log format(for now, JSON only). In that case, the console log will be printed
|
||||
in specified format like JSON.
|
||||
For example, as below:
|
||||
{"date_time":"1650918987.180175","thread_name":"#1","thread_id":"254545","level":"Trace","query_id":"","logger_name":"BaseDaemon","message":"Received signal 2","source_file":"../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()","source_line":"192"}
|
||||
To enable JSON logging support, just uncomment <formatting> tag below.
|
||||
-->
|
||||
<formatting>
|
||||
<type>json</type>
|
||||
<!--<names>
|
||||
<date_time>DATE_TIME</date_time>
|
||||
<thread_name>THREAD_NAME</thread_name>
|
||||
<thread_id>THREAD_ID</thread_id>
|
||||
<level>LEVEL</level>
|
||||
<query_id>QUERY_ID</query_id>
|
||||
<logger_name>LOGGER_NAME</logger_name>
|
||||
<message>MESSAGE</message>
|
||||
<source_file>SOURCE_FILE</source_file>
|
||||
<source_line>SOURCE_LINE</source_line>
|
||||
</names>-->
|
||||
</formatting>
|
||||
</logger>
|
||||
|
||||
</clickhouse>
|
||||
@ -0,0 +1,27 @@
|
||||
<?xml version="1.0"?>
|
||||
<clickhouse>
|
||||
<logger>
|
||||
<!-- Structured log formatting:
|
||||
You can specify log format(for now, JSON only). In that case, the console log will be printed
|
||||
in specified format like JSON.
|
||||
For example, as below:
|
||||
{"date_time":"1650918987.180175","thread_name":"#1","thread_id":"254545","level":"Trace","query_id":"","logger_name":"BaseDaemon","message":"Received signal 2","source_file":"../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()","source_line":"192"}
|
||||
To enable JSON logging support, just uncomment <formatting> tag below.
|
||||
-->
|
||||
<formatting>
|
||||
<type>json</type>
|
||||
<names>
|
||||
<date_time>DATE_TIME</date_time>
|
||||
<thread_name>THREAD_NAME</thread_name>
|
||||
<thread_id>THREAD_ID</thread_id>
|
||||
<level>LEVEL</level>
|
||||
<!--<query_id>QUERY_ID</query_id>
|
||||
<logger_name>LOGGER_NAME</logger_name>-->
|
||||
<message>MESSAGE</message>
|
||||
<source_file>SOURCE_FILE</source_file>
|
||||
<!--<source_line>SOURCE_LINE</source_line>-->
|
||||
</names>
|
||||
</formatting>
|
||||
</logger>
|
||||
|
||||
</clickhouse>
|
||||
@ -1,9 +1,12 @@
|
||||
import pytest
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
import json
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
node = cluster.add_instance("node", main_configs=["configs/config_json.xml"])
|
||||
node_all_keys = cluster.add_instance("node_all_keys", main_configs=["configs/config_all_keys_json.xml"])
|
||||
node_some_keys = cluster.add_instance("node_some_keys", main_configs=["configs/config_some_keys_json.xml"])
|
||||
node_no_keys = cluster.add_instance("node_no_keys", main_configs=["configs/config_no_keys_json.xml"])
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
@ -22,11 +25,55 @@ def is_json(log_json):
|
||||
return False
|
||||
return True
|
||||
|
||||
def validate_log_config_relation(config, logs, config_type):
|
||||
root = ET.fromstring(config)
|
||||
keys_in_config = set()
|
||||
|
||||
def test_structured_logging_json_format(start_cluster):
|
||||
node.query("SELECT 1")
|
||||
if config_type == "config_no_keys":
|
||||
keys_in_config.add("date_time")
|
||||
keys_in_config.add("thread_name")
|
||||
keys_in_config.add("thread_id")
|
||||
keys_in_config.add("level")
|
||||
keys_in_config.add("query_id")
|
||||
keys_in_config.add("logger_name")
|
||||
keys_in_config.add("message")
|
||||
keys_in_config.add("source_file")
|
||||
keys_in_config.add("source_line")
|
||||
else:
|
||||
for child in root.findall('.//names/*'):
|
||||
keys_in_config.add(child.text)
|
||||
|
||||
logs = node.grep_in_log("").split("\n")
|
||||
try:
|
||||
length = min(10, len(logs))
|
||||
for i in range(0, length):
|
||||
json_log = json.loads(logs[i])
|
||||
keys_in_log = set()
|
||||
for log_key in json_log.keys():
|
||||
keys_in_log.add(log_key)
|
||||
if log_key not in keys_in_config:
|
||||
return False
|
||||
for config_key in keys_in_config:
|
||||
if config_key not in keys_in_log:
|
||||
return False
|
||||
except ValueError as e:
|
||||
return False
|
||||
return True
|
||||
|
||||
def validate_logs(logs):
|
||||
length = min(10, len(logs))
|
||||
result = True
|
||||
for i in range(0, length):
|
||||
assert is_json(logs[i])
|
||||
result = result and is_json(logs[i])
|
||||
return result
|
||||
|
||||
def valiade_everything(config, node, config_type):
|
||||
node.query("SELECT 1")
|
||||
logs = node.grep_in_log("").split("\n")
|
||||
return validate_logs(logs) and validate_log_config_relation(config, logs, config_type)
|
||||
|
||||
def test_structured_logging_json_format2(start_cluster):
|
||||
config_all_keys = node_all_keys.exec_in_container(["cat", "/etc/clickhouse-server/config.d/config_all_keys_json.xml"])
|
||||
config_some_keys = node_some_keys.exec_in_container(["cat", "/etc/clickhouse-server/config.d/config_some_keys_json.xml"])
|
||||
config_no_keys = node_no_keys.exec_in_container(["cat", "/etc/clickhouse-server/config.d/config_no_keys_json.xml"])
|
||||
|
||||
assert valiade_everything(config_all_keys, node_all_keys, "config_all_keys") and valiade_everything(config_some_keys, node_some_keys, "config_some_keys") and valiade_everything(config_no_keys, node_no_keys, "config_no_keys")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user