ClickHouse/tests/docker_scripts/process_functional_tests_result.py

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

255 lines
8.9 KiB
Python
Raw Normal View History

2021-02-20 20:04:24 +00:00
#!/usr/bin/env python3
import os
import logging
import argparse
import csv
OK_SIGN = "[ OK "
2021-06-14 01:11:16 +00:00
FAIL_SIGN = "[ FAIL "
TIMEOUT_SIGN = "[ Timeout! "
2021-02-20 20:04:24 +00:00
UNKNOWN_SIGN = "[ UNKNOWN "
SKIPPED_SIGN = "[ SKIPPED "
HUNG_SIGN = "Found hung queries in processlist"
SERVER_DIED_SIGN = "Server died, terminating all processes"
SERVER_DIED_SIGN2 = "Server does not respond to health check"
DATABASE_SIGN = "Database: "
2021-02-20 20:04:24 +00:00
SUCCESS_FINISH_SIGNS = ["All tests have finished", "No tests were run"]
2021-03-16 16:39:31 +00:00
2021-06-16 21:20:35 +00:00
RETRIES_SIGN = "Some tests were restarted"
2023-04-12 16:46:30 +00:00
def process_test_log(log_path, broken_tests):
2021-02-20 20:04:24 +00:00
total = 0
skipped = 0
unknown = 0
failed = 0
success = 0
hung = False
server_died = False
2021-06-16 21:20:35 +00:00
retries = False
success_finish = False
2021-02-20 20:04:24 +00:00
test_results = []
test_end = True
2024-08-09 14:22:33 +00:00
with open(log_path, "r", encoding="utf-8") as test_file:
2021-02-20 20:04:24 +00:00
for line in test_file:
original_line = line
2021-02-20 20:04:24 +00:00
line = line.strip()
if any(s in line for s in SUCCESS_FINISH_SIGNS):
success_finish = True
# Ignore hung check report, since it may be quite large.
# (and may break python parser which has limit of 128KiB for each row).
2021-02-20 20:04:24 +00:00
if HUNG_SIGN in line:
hung = True
break
if SERVER_DIED_SIGN in line or SERVER_DIED_SIGN2 in line:
server_died = True
2021-06-16 21:20:35 +00:00
if RETRIES_SIGN in line:
retries = True
2021-06-14 01:11:16 +00:00
if any(
sign in line
for sign in (OK_SIGN, FAIL_SIGN, UNKNOWN_SIGN, SKIPPED_SIGN)
):
2021-02-20 20:04:24 +00:00
test_name = line.split(" ")[2].split(":")[0]
test_time = ""
try:
time_token = line.split("]")[1].strip().split()[0]
float(time_token)
test_time = time_token
except:
pass
total += 1
2021-06-14 01:11:16 +00:00
if TIMEOUT_SIGN in line:
2023-05-05 15:17:16 +00:00
if test_name in broken_tests:
success += 1
2023-05-05 17:43:54 +00:00
test_results.append((test_name, "BROKEN", test_time, []))
2023-05-05 15:17:16 +00:00
else:
failed += 1
test_results.append((test_name, "Timeout", test_time, []))
2021-06-14 01:11:16 +00:00
elif FAIL_SIGN in line:
2023-04-12 16:46:30 +00:00
if test_name in broken_tests:
success += 1
2023-05-05 17:43:54 +00:00
test_results.append((test_name, "BROKEN", test_time, []))
2023-04-12 16:46:30 +00:00
else:
failed += 1
test_results.append((test_name, "FAIL", test_time, []))
2021-02-20 20:04:24 +00:00
elif UNKNOWN_SIGN in line:
unknown += 1
2021-09-30 15:44:30 +00:00
test_results.append((test_name, "FAIL", test_time, []))
2021-02-20 20:04:24 +00:00
elif SKIPPED_SIGN in line:
skipped += 1
2021-09-30 15:44:30 +00:00
test_results.append((test_name, "SKIPPED", test_time, []))
2021-02-20 20:04:24 +00:00
else:
2023-04-12 16:46:30 +00:00
if OK_SIGN in line and test_name in broken_tests:
2023-05-05 15:17:16 +00:00
skipped += 1
2023-04-12 17:09:37 +00:00
test_results.append(
(
test_name,
2023-05-05 17:53:15 +00:00
"NOT_FAILED",
2023-04-12 17:09:37 +00:00
test_time,
["This test passed. Update analyzer_tech_debt.txt.\n"],
2023-04-12 17:09:37 +00:00
)
)
2023-04-12 16:46:30 +00:00
else:
success += int(OK_SIGN in line)
test_results.append((test_name, "OK", test_time, []))
test_end = False
elif (
len(test_results) > 0 and test_results[-1][1] == "FAIL" and not test_end
):
test_results[-1][3].append(original_line)
# Database printed after everything else in case of failures,
# so this is a stop marker for capturing test output.
#
# And it is handled after everything else to include line with database into the report.
if DATABASE_SIGN in line:
test_end = True
2021-09-30 15:44:30 +00:00
# Python does not support TSV, so we have to escape '\t' and '\n' manually
# and hope that complex escape sequences will not break anything
2021-09-30 15:44:30 +00:00
test_results = [
[
2023-02-02 00:59:49 +00:00
test[0],
test[1],
test[2],
"".join(test[3])[:8192].replace("\t", "\\t").replace("\n", "\\n"),
]
2023-02-02 00:59:49 +00:00
for test in test_results
2021-09-30 15:44:30 +00:00
]
return (
total,
skipped,
unknown,
failed,
success,
hung,
server_died,
success_finish,
retries,
2021-06-16 21:20:35 +00:00
test_results,
)
2021-09-30 15:44:30 +00:00
2021-02-20 20:04:24 +00:00
2023-04-12 16:46:30 +00:00
def process_result(result_path, broken_tests):
2021-02-20 20:04:24 +00:00
test_results = []
state = "success"
description = ""
files = os.listdir(result_path)
if files:
logging.info("Find files in result folder %s", ",".join(files))
result_path = os.path.join(result_path, "test_result.txt")
else:
result_path = None
description = "No output log"
state = "error"
if result_path and os.path.exists(result_path):
(
2024-08-09 14:22:33 +00:00
_total,
2021-06-16 21:20:35 +00:00
skipped,
unknown,
failed,
success,
hung,
server_died,
success_finish,
2021-06-16 21:20:35 +00:00
retries,
test_results,
2023-04-12 16:46:30 +00:00
) = process_test_log(result_path, broken_tests)
2024-08-04 17:18:00 +00:00
is_flaky_check = 1 < int(os.environ.get("NUM_TRIES", 1))
logging.info("Is flaky check: %s", is_flaky_check)
2021-02-20 20:04:24 +00:00
# If no tests were run (success == 0) it indicates an error (e.g. server did not start or crashed immediately)
# But it's Ok for "flaky checks" - they can contain just one test for check which is marked as skipped.
2024-08-04 17:18:00 +00:00
if failed != 0 or unknown != 0 or (success == 0 and (not is_flaky_check)):
2021-02-20 20:04:24 +00:00
state = "failure"
if hung:
description = "Some queries hung, "
state = "failure"
test_results.append(["Some queries hung", "FAIL", "0", ""])
elif server_died:
description = "Server died, "
state = "failure"
# When ClickHouse server crashes, some tests are still running
# and fail because they cannot connect to server
for result in test_results:
if result[1] == "FAIL":
result[1] = "SERVER_DIED"
test_results.append(["Server died", "FAIL", "0", ""])
elif not success_finish:
description = "Tests are not finished, "
2021-03-16 16:39:31 +00:00
state = "failure"
test_results.append(["Tests are not finished", "FAIL", "0", ""])
2021-06-16 21:20:35 +00:00
elif retries:
description = "Some tests restarted, "
test_results.append(["Some tests restarted", "SKIPPED", "0", ""])
2021-02-20 20:04:24 +00:00
else:
description = ""
2024-08-09 14:22:33 +00:00
description += f"fail: {failed}, passed: {success}"
2021-02-20 20:04:24 +00:00
if skipped != 0:
2024-08-09 14:22:33 +00:00
description += f", skipped: {skipped}"
2021-02-20 20:04:24 +00:00
if unknown != 0:
2024-08-09 14:22:33 +00:00
description += f", unknown: {unknown}"
2021-02-20 20:04:24 +00:00
else:
state = "failure"
description = "Output log doesn't exist"
test_results = []
return state, description, test_results
def write_results(results_file, status_file, results, status):
2024-08-09 14:22:33 +00:00
with open(results_file, "w", encoding="utf-8") as f:
2021-02-20 20:04:24 +00:00
out = csv.writer(f, delimiter="\t")
out.writerows(results)
2024-08-09 14:22:33 +00:00
with open(status_file, "w", encoding="utf-8") as f:
2021-02-20 20:04:24 +00:00
out = csv.writer(f, delimiter="\t")
out.writerow(status)
2021-02-22 14:43:06 +00:00
2021-02-20 20:04:24 +00:00
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
parser = argparse.ArgumentParser(
description="ClickHouse script for parsing results of functional tests"
)
parser.add_argument("--in-results-dir", default="/test_output/")
parser.add_argument("--out-results-file", default="/test_output/test_results.tsv")
parser.add_argument("--out-status-file", default="/test_output/check_status.tsv")
2024-08-09 14:22:33 +00:00
parser.add_argument("--broken-tests", default="/repo/tests/analyzer_tech_debt.txt")
2021-02-20 20:04:24 +00:00
args = parser.parse_args()
2024-08-09 14:22:33 +00:00
broken_tests = []
2023-04-18 00:12:30 +00:00
if os.path.exists(args.broken_tests):
2024-08-09 14:22:33 +00:00
print(f"File {args.broken_tests} with broken tests found")
with open(args.broken_tests, encoding="utf-8") as f:
2023-04-18 13:53:13 +00:00
broken_tests = f.read().splitlines()
2024-08-09 14:22:33 +00:00
print(f"Broken tests in the list: {len(broken_tests)}")
2023-04-12 16:46:30 +00:00
state, description, test_results = process_result(args.in_results_dir, broken_tests)
2021-02-20 20:04:24 +00:00
logging.info("Result parsed")
status = (state, description)
def test_result_comparator(item):
# sort by status then by check name
order = {
"FAIL": 0,
"SERVER_DIED": 1,
"Timeout": 2,
"NOT_FAILED": 3,
"BROKEN": 4,
"OK": 5,
"SKIPPED": 6,
}
return order.get(item[1], 10), str(item[0]), item[1]
test_results.sort(key=test_result_comparator)
2021-02-20 20:04:24 +00:00
write_results(args.out_results_file, args.out_status_file, test_results, status)
logging.info("Result written")