ClickHouse/docker/test/performance-comparison/report.py

756 lines
22 KiB
Python
Raw Normal View History

2020-01-23 17:48:26 +00:00
#!/usr/bin/python3
2020-04-02 18:44:58 +00:00
import argparse
2020-03-24 17:33:18 +00:00
import ast
2020-01-23 17:48:26 +00:00
import collections
import csv
2020-02-27 17:57:08 +00:00
import itertools
import json
2020-01-23 17:48:26 +00:00
import os
2020-06-29 14:09:51 +00:00
import os.path
2020-06-22 12:22:09 +00:00
import pprint
2020-01-23 17:48:26 +00:00
import sys
2020-02-26 16:18:58 +00:00
import traceback
2020-01-23 17:48:26 +00:00
parser = argparse.ArgumentParser(description="Create performance test report")
parser.add_argument(
"--report",
default="main",
choices=["main", "all-queries"],
help="Which report to build",
)
parser.add_argument("--no-tests-run", action="store_true", default=False)
2020-04-02 18:44:58 +00:00
args = parser.parse_args()
2020-08-05 15:46:05 +00:00
tables = []
2020-08-05 16:07:37 +00:00
errors_explained = []
2020-02-26 16:18:58 +00:00
report_errors = []
2020-02-27 19:43:43 +00:00
error_tests = 0
slow_average_tests = 0
faster_queries = 0
slower_queries = 0
unstable_queries = 0
2020-03-24 17:33:18 +00:00
very_unstable_queries = 0
unstable_backward_incompatible_queries = 0
2020-02-26 16:18:58 +00:00
2020-04-29 17:26:28 +00:00
# max seconds to run one query by itself, not counting preparation
allowed_single_run_time = 2
color_bad = "#ffb0c0"
color_good = "#b0d050"
2020-04-02 18:44:58 +00:00
header_template = """
2020-01-23 17:48:26 +00:00
<!DOCTYPE html>
2020-08-28 22:13:46 +00:00
<html lang="en">
2020-08-28 22:46:04 +00:00
<style>
2020-07-31 19:58:18 +00:00
body {{
font-family: "DejaVu Sans", "Noto Sans", Arial, sans-serif;
2020-07-31 19:58:18 +00:00
background: #EEE;
}}
2020-01-23 17:48:26 +00:00
a {{ color: #06F; text-decoration: none; }}
2020-07-31 19:58:18 +00:00
2020-01-23 17:48:26 +00:00
a:hover, a:active {{ color: #F40; text-decoration: underline; }}
2020-07-31 19:58:18 +00:00
2020-05-09 15:50:28 +00:00
.main {{ margin: auto; max-width: 95%; }}
2020-02-27 17:57:08 +00:00
2020-07-31 19:58:18 +00:00
p.links a {{
padding: 5px; margin: 3px; background: #FFF; line-height: 2;
white-space: nowrap;
box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1);
}}
.cancela,.cancela:link,.cancela:visited,.cancela:hover,
.cancela:focus,.cancela:active {{
2020-02-10 16:34:07 +00:00
color: inherit;
text-decoration: none;
}}
2020-07-31 19:58:18 +00:00
table {{
border: none;
border-spacing: 0px;
line-height: 1.5;
box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1);
text-align: left;
}}
th, td {{
border: none;
padding: 5px;
vertical-align: top;
background-color: #FFF;
font-family: sans-serif;
}}
th {{
border-bottom: 2px solid black;
}}
tr:nth-child(odd) td {{filter: brightness(90%);}}
.unexpected-query-duration tr :nth-child(2),
.unexpected-query-duration tr :nth-child(3),
.unexpected-query-duration tr :nth-child(5),
2020-07-31 19:58:18 +00:00
.all-query-times tr :nth-child(1),
.all-query-times tr :nth-child(2),
.all-query-times tr :nth-child(3),
.all-query-times tr :nth-child(4),
.all-query-times tr :nth-child(5),
.all-query-times tr :nth-child(7),
.changes-in-performance tr :nth-child(1),
.changes-in-performance tr :nth-child(2),
.changes-in-performance tr :nth-child(3),
.changes-in-performance tr :nth-child(4),
.changes-in-performance tr :nth-child(5),
.changes-in-performance tr :nth-child(7),
.unstable-queries tr :nth-child(1),
.unstable-queries tr :nth-child(2),
.unstable-queries tr :nth-child(3),
.unstable-queries tr :nth-child(4),
.unstable-queries tr :nth-child(6),
.test-performance-changes tr :nth-child(2),
.test-performance-changes tr :nth-child(3),
.test-performance-changes tr :nth-child(4),
.test-performance-changes tr :nth-child(5),
.test-performance-changes tr :nth-child(6),
.test-times tr :nth-child(2),
.test-times tr :nth-child(3),
.test-times tr :nth-child(4),
.test-times tr :nth-child(5),
.test-times tr :nth-child(6),
.test-times tr :nth-child(7),
.concurrent-benchmarks tr :nth-child(2),
.concurrent-benchmarks tr :nth-child(3),
.concurrent-benchmarks tr :nth-child(4),
.concurrent-benchmarks tr :nth-child(5),
.metric-changes tr :nth-child(2),
.metric-changes tr :nth-child(3),
.metric-changes tr :nth-child(4),
.metric-changes tr :nth-child(5)
{{ text-align: right; }}
2020-01-23 17:48:26 +00:00
</style>
2020-02-26 16:18:58 +00:00
<title>Clickhouse performance comparison</title>
2020-01-23 17:48:26 +00:00
</head>
<body>
<div class="main">
2020-02-26 16:18:58 +00:00
<h1>ClickHouse performance comparison</h1>
2020-04-02 18:44:58 +00:00
"""
2020-02-26 16:18:58 +00:00
2020-02-10 16:34:07 +00:00
table_anchor = 0
row_anchor = 0
2020-08-05 15:46:05 +00:00
def currentTableAnchor():
global table_anchor
return f"{table_anchor}"
2020-08-05 15:46:05 +00:00
def newTableAnchor():
2020-02-10 16:34:07 +00:00
global table_anchor
table_anchor += 1
2020-08-05 15:46:05 +00:00
return currentTableAnchor()
2020-08-05 15:46:05 +00:00
def currentRowAnchor():
global row_anchor
global table_anchor
return f"{table_anchor}.{row_anchor}"
2020-02-10 16:34:07 +00:00
def nextRowAnchor():
2020-08-05 15:46:05 +00:00
global row_anchor
global table_anchor
return f"{table_anchor}.{row_anchor + 1}"
2020-08-05 15:46:05 +00:00
def advanceRowAnchor():
2020-02-10 16:34:07 +00:00
global row_anchor
global table_anchor
row_anchor += 1
2020-08-05 15:46:05 +00:00
return currentRowAnchor()
2020-02-10 16:34:07 +00:00
2020-08-07 01:25:45 +00:00
def tr(x, anchor=None):
# return '<tr onclick="location.href=\'#{a}\'" id={a}>{x}</tr>'.format(a=a, x=str(x))
2020-08-07 01:25:45 +00:00
anchor = anchor if anchor else advanceRowAnchor()
return f"<tr id={anchor}>{x}</tr>"
2020-01-23 17:48:26 +00:00
def td(value, cell_attributes=""):
return "<td {cell_attributes}>{value}</td>".format(
cell_attributes=cell_attributes, value=value
)
2020-01-23 17:48:26 +00:00
def th(value, cell_attributes=""):
return "<th {cell_attributes}>{value}</th>".format(
cell_attributes=cell_attributes, value=value
)
def tableRow(cell_values, cell_attributes=[], anchor=None):
2020-08-07 01:25:45 +00:00
return tr(
"".join(
[
td(v, a)
for v, a in itertools.zip_longest(
cell_values, cell_attributes, fillvalue=""
)
if a is not None and v is not None
]
),
anchor,
)
def tableHeader(cell_values, cell_attributes=[]):
2020-09-23 20:45:39 +00:00
return tr(
"".join(
[
th(v, a)
for v, a in itertools.zip_longest(
cell_values, cell_attributes, fillvalue=""
)
if a is not None and v is not None
]
)
)
2020-01-23 17:48:26 +00:00
2020-02-26 16:18:58 +00:00
def tableStart(title):
cls = "-".join(title.lower().split(" ")[:3])
2020-08-05 15:46:05 +00:00
global table_anchor
table_anchor = cls
anchor = currentTableAnchor()
help_anchor = "-".join(title.lower().split(" "))
return f"""
<h2 id="{anchor}">
<a class="cancela" href="#{anchor}">{title}</a>
<a class="cancela" href="https://github.com/ClickHouse/ClickHouse/tree/master/docker/test/performance-comparison#{help_anchor}"><sup style="color: #888">?</sup></a>
</h2>
<table class="{cls}">
"""
2020-02-26 16:18:58 +00:00
2020-02-26 16:18:58 +00:00
def tableEnd():
return "</table>"
2020-02-26 16:18:58 +00:00
2020-02-27 17:57:08 +00:00
def tsvRows(n):
2020-02-26 16:18:58 +00:00
try:
with open(n, encoding="utf-8") as fd:
result = []
for row in csv.reader(fd, delimiter="\t", quoting=csv.QUOTE_NONE):
new_row = []
for e in row:
2021-11-25 12:37:00 +00:00
# The first one .encode('utf-8').decode('unicode-escape') decodes the escape characters from the strings.
# The second one (encode('latin1').decode('utf-8')) fixes the changes with unicode vs utf-8 chars, so
# 'Чем зÐ<C2B7>нимаеÑ<C2B5>ЬÑ<C2AC>Ñ<EFBFBD>' is transformed back into 'Чем зАнимаешЬся'.
new_row.append(
e.encode("utf-8")
.decode("unicode-escape")
.encode("latin1")
.decode("utf-8")
)
result.append(new_row)
return result
2020-02-26 16:18:58 +00:00
except:
report_errors.append(traceback.format_exception_only(*sys.exc_info()[:2])[-1])
2020-02-26 16:18:58 +00:00
pass
return []
2020-02-27 17:57:08 +00:00
def htmlRows(n):
rawRows = tsvRows(n)
result = ""
2020-02-26 16:18:58 +00:00
for row in rawRows:
result += tableRow(row)
2020-01-23 17:48:26 +00:00
return result
2020-08-05 15:46:05 +00:00
def addSimpleTable(caption, columns, rows, pos=None):
global tables
text = ""
2020-02-27 17:57:08 +00:00
if not rows:
return
2020-08-05 15:46:05 +00:00
text += tableStart(caption)
text += tableHeader(columns)
2020-02-27 17:57:08 +00:00
for row in rows:
2020-08-05 15:46:05 +00:00
text += tableRow(row)
text += tableEnd()
tables.insert(pos if pos else len(tables), text)
2020-02-27 17:57:08 +00:00
2020-08-05 15:46:05 +00:00
def add_tested_commits():
2020-04-30 08:36:33 +00:00
global report_errors
try:
addSimpleTable(
"Tested Commits",
["Old", "New"],
[
[
"<pre>{}</pre>".format(x)
for x in [
open("left-commit.txt").read(),
open("right-commit.txt").read(),
]
]
],
)
2020-04-30 08:36:33 +00:00
except:
# Don't fail if no commit info -- maybe it's a manual run.
report_errors.append(traceback.format_exception_only(*sys.exc_info()[:2])[-1])
2020-04-30 08:36:33 +00:00
pass
2020-08-05 15:46:05 +00:00
def add_report_errors():
global tables
2020-04-30 08:36:33 +00:00
global report_errors
# Add the errors reported by various steps of comparison script
try:
report_errors += [l.strip() for l in open("report/errors.log")]
2020-04-30 08:36:33 +00:00
except:
report_errors.append(traceback.format_exception_only(*sys.exc_info()[:2])[-1])
2020-04-30 08:36:33 +00:00
pass
2020-08-05 15:46:05 +00:00
if not report_errors:
return
text = tableStart("Errors while Building the Report")
text += tableHeader(["Error"])
2020-08-05 15:46:05 +00:00
for x in report_errors:
text += tableRow([x])
text += tableEnd()
# Insert after Tested Commits
tables.insert(1, text)
errors_explained.append(
[
f'<a href="#{currentTableAnchor()}">There were some errors while building the report</a>'
]
)
2020-08-05 15:46:05 +00:00
def add_errors_explained():
2020-08-06 13:28:51 +00:00
if not errors_explained:
return
2020-08-07 01:25:45 +00:00
text = '<a name="fail1"/>'
text += tableStart("Error Summary")
text += tableHeader(["Description"])
2020-08-06 13:28:51 +00:00
for row in errors_explained:
text += tableRow(row)
text += tableEnd()
2020-08-05 15:46:05 +00:00
global tables
2020-08-06 13:28:51 +00:00
tables.insert(1, text)
2020-08-05 15:46:05 +00:00
2020-04-30 08:36:33 +00:00
if args.report == "main":
2020-10-02 16:54:07 +00:00
print((header_template.format()))
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_tested_commits()
def print_status(status, message):
print(
(
"""
<!--status: {status}-->
<!--message: {message}-->
""".format(
status=status, message=message
)
)
)
if args.no_tests_run:
for t in tables:
print(t)
print(
"<h2>No tests to run. Only changed tests were run, but all changed tests are from another batch.</h2>"
)
print(
f"""
</div>
{os.getenv("CHPC_ADD_REPORT_LINKS") or ''}
</body>
</html>
"""
)
# Why failure? Because otherwise we will not notice if we have a bug that leads to 0 tests being run
print_status("failure", "No tests changed, nothing to run")
exit(0)
run_error_rows = tsvRows("run-errors.tsv")
2020-05-22 02:51:26 +00:00
error_tests += len(run_error_rows)
addSimpleTable("Run Errors", ["Test", "Error"], run_error_rows)
2020-08-05 15:46:05 +00:00
if run_error_rows:
errors_explained.append(
[
f'<a href="#{currentTableAnchor()}">There were some errors while running the tests</a>'
]
)
2020-05-22 02:51:26 +00:00
def add_backward_incompatible():
rows = tsvRows("report/partial-queries-report.tsv")
if not rows:
return
2020-08-05 15:46:05 +00:00
global unstable_backward_incompatible_queries, slow_average_tests, tables
2023-01-22 19:01:55 +00:00
text = tableStart("Backward-incompatible queries")
columns = ["Median time, s", "Relative time variance", "Test", "#", "Query"]
2020-08-05 15:46:05 +00:00
text += tableHeader(columns)
attrs = ["" for c in columns]
for row in rows:
anchor = f"{currentTableAnchor()}.{row[2]}.{row[3]}"
if float(row[1]) > 0.10:
attrs[1] = f'style="background: {color_bad}"'
unstable_backward_incompatible_queries += 1
errors_explained.append(
[
f"<a href=\"#{anchor}\">The query no. {row[3]} of test '{row[2]}' has excessive variance of run time. Keep it below 10%</a>"
]
)
else:
attrs[1] = ""
if float(row[0]) > allowed_single_run_time:
attrs[0] = f'style="background: {color_bad}"'
errors_explained.append(
[
f'<a href="#{anchor}">The query no. {row[3]} of test \'{row[2]}\' is taking too long to run. Keep the run time below {allowed_single_run_time} seconds"</a>'
]
)
slow_average_tests += 1
else:
attrs[0] = ""
2020-08-07 01:25:45 +00:00
text += tableRow(row, attrs, anchor)
2020-08-05 15:46:05 +00:00
text += tableEnd()
tables.append(text)
add_backward_incompatible()
2020-08-05 15:46:05 +00:00
def add_changes():
rows = tsvRows("report/changed-perf.tsv")
2020-04-02 18:44:58 +00:00
if not rows:
return
2020-08-05 15:46:05 +00:00
global faster_queries, slower_queries, tables
2020-04-02 18:44:58 +00:00
text = tableStart("Changes in Performance")
2020-04-02 18:44:58 +00:00
columns = [
"Old,&nbsp;s", # 0
"New,&nbsp;s", # 1
"Ratio of speedup&nbsp;(-) or slowdown&nbsp;(+)", # 2
"Relative difference (new&nbsp;&minus;&nbsp;old) / old", # 3
"p&nbsp;<&nbsp;0.01 threshold", # 4
"", # Failed # 5
"Test", # 6
"#", # 7
"Query", # 8
]
attrs = ["" for c in columns]
2020-08-01 10:10:36 +00:00
attrs[5] = None
2020-09-23 20:45:39 +00:00
text += tableHeader(columns, attrs)
2020-04-02 18:44:58 +00:00
for row in rows:
anchor = f"{currentTableAnchor()}.{row[6]}.{row[7]}"
if int(row[5]):
if float(row[3]) < 0.0:
2020-04-21 18:46:45 +00:00
faster_queries += 1
attrs[2] = attrs[3] = f'style="background: {color_good}"'
2020-04-21 18:46:45 +00:00
else:
slower_queries += 1
attrs[2] = attrs[3] = f'style="background: {color_bad}"'
errors_explained.append(
[
f"<a href=\"#{anchor}\">The query no. {row[7]} of test '{row[6]}' has slowed down</a>"
]
)
2020-04-28 07:45:35 +00:00
else:
attrs[2] = attrs[3] = ""
2020-04-02 18:44:58 +00:00
2020-08-07 01:25:45 +00:00
text += tableRow(row, attrs, anchor)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text += tableEnd()
tables.append(text)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_changes()
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
def add_unstable_queries():
global unstable_queries, very_unstable_queries, tables
2020-04-02 18:44:58 +00:00
unstable_rows = tsvRows("report/unstable-queries.tsv")
2020-04-02 18:44:58 +00:00
if not unstable_rows:
return
unstable_queries += len(unstable_rows)
columns = [
"Old,&nbsp;s", # 0
"New,&nbsp;s", # 1
"Relative difference (new&nbsp;-&nbsp;old)/old", # 2
"p&nbsp;&lt;&nbsp;0.01 threshold", # 3
"", # Failed #4
"Test", # 5
"#", # 6
"Query", # 7
2020-02-27 17:57:08 +00:00
]
attrs = ["" for c in columns]
2020-09-23 20:45:39 +00:00
attrs[4] = None
2020-02-27 17:57:08 +00:00
text = tableStart("Unstable Queries")
2020-09-23 20:45:39 +00:00
text += tableHeader(columns, attrs)
2020-04-02 18:44:58 +00:00
for r in unstable_rows:
anchor = f"{currentTableAnchor()}.{r[5]}.{r[6]}"
2020-04-28 07:45:35 +00:00
if int(r[4]):
2020-04-02 18:44:58 +00:00
very_unstable_queries += 1
attrs[3] = f'style="background: {color_bad}"'
2020-04-02 18:44:58 +00:00
else:
attrs[3] = ""
2021-05-13 09:16:43 +00:00
# Just don't add the slightly unstable queries we don't consider
# errors. It's not clear what the user should do with them.
continue
2020-04-02 18:44:58 +00:00
2020-08-07 01:25:45 +00:00
text += tableRow(r, attrs, anchor)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text += tableEnd()
# Don't add an empty table.
if very_unstable_queries:
tables.append(text)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_unstable_queries()
2020-04-02 18:44:58 +00:00
skipped_tests_rows = tsvRows("analyze/skipped-tests.tsv")
addSimpleTable("Skipped Tests", ["Test", "Reason"], skipped_tests_rows)
addSimpleTable(
"Test Performance Changes",
[
"Test",
"Ratio of speedup&nbsp;(-) or slowdown&nbsp;(+)",
"Queries",
"Total not OK",
"Changed perf",
"Unstable",
],
tsvRows("report/test-perf-changes.tsv"),
)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
def add_test_times():
global slow_average_tests, tables
rows = tsvRows("report/test-times.tsv")
2020-04-02 18:44:58 +00:00
if not rows:
return
columns = [
"Test", # 0
"Wall clock time, entire test,&nbsp;s", # 1
"Total client time for measured query runs,&nbsp;s", # 2
"Queries", # 3
"Longest query, total for measured runs,&nbsp;s", # 4
"Wall clock time per query,&nbsp;s", # 5
"Shortest query, total for measured runs,&nbsp;s", # 6
"", # Runs #7
]
attrs = ["" for c in columns]
2020-09-23 07:28:28 +00:00
attrs[7] = None
2020-04-02 18:44:58 +00:00
text = tableStart("Test Times")
2020-09-23 20:45:39 +00:00
text += tableHeader(columns, attrs)
2020-05-09 15:58:47 +00:00
allowed_average_run_time = 3.75 # 60 seconds per test at (7 + 1) * 2 runs
2020-04-02 18:44:58 +00:00
for r in rows:
anchor = f"{currentTableAnchor()}.{r[0]}"
total_runs = (int(r[7]) + 1) * 2 # one prewarm run, two servers
if r[0] != "Total" and float(r[5]) > allowed_average_run_time * total_runs:
2020-04-02 18:44:58 +00:00
# FIXME should be 15s max -- investigate parallel_insert
slow_average_tests += 1
2020-09-03 01:42:25 +00:00
attrs[5] = f'style="background: {color_bad}"'
errors_explained.append(
[
f"<a href=\"#{anchor}\">The test '{r[0]}' is too slow to run as a whole. Investigate whether the create and fill queries can be sped up"
]
)
2020-04-02 18:44:58 +00:00
else:
attrs[5] = ""
2020-04-02 18:44:58 +00:00
if r[0] != "Total" and float(r[4]) > allowed_single_run_time * total_runs:
2020-04-02 18:44:58 +00:00
slow_average_tests += 1
2020-09-03 01:42:25 +00:00
attrs[4] = f'style="background: {color_bad}"'
errors_explained.append(
[
f"<a href=\"./all-queries.html#all-query-times.{r[0]}.0\">Some query of the test '{r[0]}' is too slow to run. See the all queries report"
]
)
2020-04-02 18:44:58 +00:00
else:
attrs[4] = ""
2020-04-02 18:44:58 +00:00
2020-09-01 19:05:57 +00:00
text += tableRow(r, attrs, anchor)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text += tableEnd()
tables.append(text)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_test_times()
2020-04-02 18:44:58 +00:00
addSimpleTable(
"Metric Changes",
[
"Metric",
"Old median value",
"New median value",
"Relative difference",
"Times difference",
],
tsvRows("metrics/changes.tsv"),
)
2020-06-25 20:19:27 +00:00
2020-08-05 15:46:05 +00:00
add_report_errors()
add_errors_explained()
for t in tables:
print(t)
2020-04-02 18:44:58 +00:00
print(
f"""
2020-08-28 22:46:04 +00:00
</div>
2020-04-02 18:44:58 +00:00
<p class="links">
<a href="all-queries.html">All queries</a>
<a href="compare.log">Log</a>
<a href="output.7z">Test output</a>
2021-04-23 13:47:13 +00:00
{os.getenv("CHPC_ADD_REPORT_LINKS") or ''}
2020-04-02 18:44:58 +00:00
</p>
</body>
</html>
"""
)
2020-04-02 18:44:58 +00:00
status = "success"
message = "See the report"
2020-04-02 18:44:58 +00:00
message_array = []
if slow_average_tests:
status = "failure"
message_array.append(str(slow_average_tests) + " too long")
2020-04-02 18:44:58 +00:00
if faster_queries:
message_array.append(str(faster_queries) + " faster")
2020-04-02 18:44:58 +00:00
if slower_queries:
2023-05-30 18:53:45 +00:00
# This threshold should be synchronized with the value in https://github.com/ClickHouse/ClickHouse/blob/master/tests/ci/performance_comparison_check.py#L225
# False positives rate should be < 1%: https://shorturl.at/CDEK8
if slower_queries > 5:
status = "failure"
message_array.append(str(slower_queries) + " slower")
2020-04-02 18:44:58 +00:00
if unstable_backward_incompatible_queries:
very_unstable_queries += unstable_backward_incompatible_queries
status = "failure"
2021-05-12 22:32:53 +00:00
# Don't show mildly unstable queries, only the very unstable ones we
# treat as errors.
if very_unstable_queries:
message_array.append(str(very_unstable_queries) + " unstable")
# FIXME: uncomment the following lines when tests are stable and
# reliable
# if very_unstable_queries > 5:
# error_tests += very_unstable_queries
# status = "failure"
#
# error_tests += slow_average_tests
# FIXME: until here
2020-04-02 18:44:58 +00:00
if error_tests:
status = "failure"
message_array.insert(0, str(error_tests) + " errors")
2020-04-02 18:44:58 +00:00
if message_array:
message = ", ".join(message_array)
2020-04-02 18:44:58 +00:00
if report_errors:
status = "failure"
message = "Errors while building the report."
2020-04-02 18:44:58 +00:00
print_status(status, message)
2020-04-02 18:44:58 +00:00
elif args.report == "all-queries":
2020-10-02 16:54:07 +00:00
print((header_template.format()))
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_tested_commits()
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
def add_all_queries():
rows = tsvRows("report/all-queries.tsv")
2020-04-02 18:44:58 +00:00
if not rows:
return
columns = [
"", # Changed #0
"", # Unstable #1
"Old,&nbsp;s", # 2
"New,&nbsp;s", # 3
"Ratio of speedup&nbsp;(-) or slowdown&nbsp;(+)", # 4
"Relative difference (new&nbsp;&minus;&nbsp;old) / old", # 5
"p&nbsp;&lt;&nbsp;0.01 threshold", # 6
"Test", # 7
"#", # 8
"Query", # 9
]
attrs = ["" for c in columns]
2020-04-28 07:45:35 +00:00
attrs[0] = None
attrs[1] = None
2020-09-23 20:45:39 +00:00
text = tableStart("All Query Times")
2020-09-23 20:45:39 +00:00
text += tableHeader(columns, attrs)
2020-04-02 18:44:58 +00:00
for r in rows:
anchor = f"{currentTableAnchor()}.{r[7]}.{r[8]}"
2020-04-28 07:45:35 +00:00
if int(r[1]):
attrs[6] = f'style="background: {color_bad}"'
2020-04-21 18:46:45 +00:00
else:
attrs[6] = ""
2020-04-21 18:46:45 +00:00
2020-04-28 07:45:35 +00:00
if int(r[0]):
if float(r[5]) > 0.0:
attrs[4] = attrs[5] = f'style="background: {color_bad}"'
2020-04-21 18:46:45 +00:00
else:
attrs[4] = attrs[5] = f'style="background: {color_good}"'
2020-04-02 18:44:58 +00:00
else:
attrs[4] = attrs[5] = ""
2020-04-02 18:44:58 +00:00
2020-04-29 17:26:28 +00:00
if (float(r[2]) + float(r[3])) / 2 > allowed_single_run_time:
attrs[2] = f'style="background: {color_bad}"'
attrs[3] = f'style="background: {color_bad}"'
2020-04-29 17:26:28 +00:00
else:
attrs[2] = ""
attrs[3] = ""
2020-04-29 17:26:28 +00:00
2020-08-07 01:25:45 +00:00
text += tableRow(r, attrs, anchor)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text += tableEnd()
tables.append(text)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_all_queries()
add_report_errors()
for t in tables:
print(t)
2020-04-30 08:36:33 +00:00
print(
f"""
2020-08-28 22:46:04 +00:00
</div>
2020-04-02 18:44:58 +00:00
<p class="links">
<a href="report.html">Main report</a>
<a href="compare.log">Log</a>
<a href="output.7z">Test output</a>
2021-04-23 13:47:13 +00:00
{os.getenv("CHPC_ADD_REPORT_LINKS") or ''}
2020-04-02 18:44:58 +00:00
</p>
</body>
</html>
"""
)