ClickHouse/docker/test/performance-comparison/report.py

669 lines
21 KiB
Python
Raw Normal View History

2020-01-23 17:48:26 +00:00
#!/usr/bin/python3
2020-04-02 18:44:58 +00:00
import argparse
2020-03-24 17:33:18 +00:00
import ast
2020-01-23 17:48:26 +00:00
import collections
import csv
2020-02-27 17:57:08 +00:00
import itertools
import json
2020-01-23 17:48:26 +00:00
import os
2020-06-29 14:09:51 +00:00
import os.path
2020-06-22 12:22:09 +00:00
import pprint
2020-01-23 17:48:26 +00:00
import sys
2020-02-26 16:18:58 +00:00
import traceback
2020-01-23 17:48:26 +00:00
2020-04-02 18:44:58 +00:00
parser = argparse.ArgumentParser(description='Create performance test report')
parser.add_argument('--report', default='main', choices=['main', 'all-queries'],
help='Which report to build')
args = parser.parse_args()
2020-08-05 15:46:05 +00:00
tables = []
2020-08-05 16:07:37 +00:00
errors_explained = []
2020-02-26 16:18:58 +00:00
report_errors = []
2020-02-27 19:43:43 +00:00
error_tests = 0
slow_average_tests = 0
faster_queries = 0
slower_queries = 0
unstable_queries = 0
2020-03-24 17:33:18 +00:00
very_unstable_queries = 0
unstable_partial_queries = 0
2020-02-26 16:18:58 +00:00
2020-04-29 17:26:28 +00:00
# max seconds to run one query by itself, not counting preparation
allowed_single_run_time = 2
color_bad='#ffb0c0'
color_good='#b0d050'
2020-04-02 18:44:58 +00:00
header_template = """
2020-01-23 17:48:26 +00:00
<!DOCTYPE html>
2020-08-28 22:13:46 +00:00
<html lang="en">
2020-08-28 22:46:04 +00:00
<link rel="preload" as="font" href="https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2" type="font/woff2" crossorigin="anonymous"/>
<style>
2020-01-23 17:48:26 +00:00
@font-face {{
font-family:'Yandex Sans Display Web';
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot);
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot?#iefix) format('embedded-opentype'),
url(https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2) format('woff2'),
url(https://yastatic.net/adv-www/_/v2Sve_obH3rKm6rKrtSQpf-eB7U.woff) format('woff'),
url(https://yastatic.net/adv-www/_/PzD8hWLMunow5i3RfJ6WQJAL7aI.ttf) format('truetype'),
url(https://yastatic.net/adv-www/_/lF_KG5g4tpQNlYIgA0e77fBSZ5s.svg#YandexSansDisplayWeb-Regular) format('svg');
font-weight:400;
font-style:normal;
2020-08-28 22:13:46 +00:00
font-stretch:normal;
font-display: swap;
2020-01-23 17:48:26 +00:00
}}
2020-07-31 19:58:18 +00:00
body {{
font-family: "Yandex Sans Display Web", Arial, sans-serif;
background: #EEE;
}}
2020-01-23 17:48:26 +00:00
a {{ color: #06F; text-decoration: none; }}
2020-07-31 19:58:18 +00:00
2020-01-23 17:48:26 +00:00
a:hover, a:active {{ color: #F40; text-decoration: underline; }}
2020-07-31 19:58:18 +00:00
2020-05-09 15:50:28 +00:00
.main {{ margin: auto; max-width: 95%; }}
2020-02-27 17:57:08 +00:00
2020-07-31 19:58:18 +00:00
p.links a {{
padding: 5px; margin: 3px; background: #FFF; line-height: 2;
white-space: nowrap;
box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1);
}}
.cancela,.cancela:link,.cancela:visited,.cancela:hover,
.cancela:focus,.cancela:active {{
2020-02-10 16:34:07 +00:00
color: inherit;
text-decoration: none;
}}
2020-07-31 19:58:18 +00:00
table {{
border: none;
border-spacing: 0px;
line-height: 1.5;
box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1);
text-align: left;
}}
th, td {{
border: none;
padding: 5px;
vertical-align: top;
background-color: #FFF;
font-family: sans-serif;
}}
th {{
border-bottom: 2px solid black;
}}
tr:nth-child(odd) td {{filter: brightness(90%);}}
.unexpected-query-duration tr :nth-child(2),
.unexpected-query-duration tr :nth-child(3),
.unexpected-query-duration tr :nth-child(5),
2020-07-31 19:58:18 +00:00
.all-query-times tr :nth-child(1),
.all-query-times tr :nth-child(2),
.all-query-times tr :nth-child(3),
.all-query-times tr :nth-child(4),
.all-query-times tr :nth-child(5),
.all-query-times tr :nth-child(7),
.changes-in-performance tr :nth-child(1),
.changes-in-performance tr :nth-child(2),
.changes-in-performance tr :nth-child(3),
.changes-in-performance tr :nth-child(4),
.changes-in-performance tr :nth-child(5),
.changes-in-performance tr :nth-child(7),
.unstable-queries tr :nth-child(1),
.unstable-queries tr :nth-child(2),
.unstable-queries tr :nth-child(3),
.unstable-queries tr :nth-child(4),
.unstable-queries tr :nth-child(6),
.test-performance-changes tr :nth-child(2),
.test-performance-changes tr :nth-child(3),
.test-performance-changes tr :nth-child(4),
.test-performance-changes tr :nth-child(5),
.test-performance-changes tr :nth-child(6),
.test-times tr :nth-child(2),
.test-times tr :nth-child(3),
.test-times tr :nth-child(4),
.test-times tr :nth-child(5),
.test-times tr :nth-child(6),
.test-times tr :nth-child(7),
.concurrent-benchmarks tr :nth-child(2),
.concurrent-benchmarks tr :nth-child(3),
.concurrent-benchmarks tr :nth-child(4),
.concurrent-benchmarks tr :nth-child(5),
.metric-changes tr :nth-child(2),
.metric-changes tr :nth-child(3),
.metric-changes tr :nth-child(4),
.metric-changes tr :nth-child(5)
{{ text-align: right; }}
2020-01-23 17:48:26 +00:00
</style>
2020-02-26 16:18:58 +00:00
<title>Clickhouse performance comparison</title>
2020-01-23 17:48:26 +00:00
</head>
<body>
<div class="main">
2020-02-26 16:18:58 +00:00
<h1>ClickHouse performance comparison</h1>
2020-04-02 18:44:58 +00:00
"""
2020-02-26 16:18:58 +00:00
2020-02-10 16:34:07 +00:00
table_anchor = 0
row_anchor = 0
2020-08-05 15:46:05 +00:00
def currentTableAnchor():
global table_anchor
return f'{table_anchor}'
def newTableAnchor():
2020-02-10 16:34:07 +00:00
global table_anchor
table_anchor += 1
2020-08-05 15:46:05 +00:00
return currentTableAnchor()
def currentRowAnchor():
global row_anchor
global table_anchor
return f'{table_anchor}.{row_anchor}'
2020-02-10 16:34:07 +00:00
def nextRowAnchor():
2020-08-05 15:46:05 +00:00
global row_anchor
global table_anchor
return f'{table_anchor}.{row_anchor + 1}'
def advanceRowAnchor():
2020-02-10 16:34:07 +00:00
global row_anchor
global table_anchor
row_anchor += 1
2020-08-05 15:46:05 +00:00
return currentRowAnchor()
2020-02-10 16:34:07 +00:00
2020-08-07 01:25:45 +00:00
def tr(x, anchor=None):
2020-02-11 20:00:53 +00:00
#return '<tr onclick="location.href=\'#{a}\'" id={a}>{x}</tr>'.format(a=a, x=str(x))
2020-08-07 01:25:45 +00:00
anchor = anchor if anchor else advanceRowAnchor()
return f'<tr id={anchor}>{x}</tr>'
2020-01-23 17:48:26 +00:00
2020-02-27 17:57:08 +00:00
def td(value, cell_attributes = ''):
return '<td {cell_attributes}>{value}</td>'.format(
cell_attributes = cell_attributes,
2020-02-26 16:18:58 +00:00
value = value)
2020-01-23 17:48:26 +00:00
2020-09-23 20:45:39 +00:00
def th(value, cell_attributes = ''):
return '<th {cell_attributes}>{value}</th>'.format(
cell_attributes = cell_attributes,
value = value)
2020-01-23 17:48:26 +00:00
2020-08-07 01:25:45 +00:00
def tableRow(cell_values, cell_attributes = [], anchor=None):
return tr(
''.join([td(v, a)
for v, a in itertools.zip_longest(
cell_values, cell_attributes,
fillvalue = '')
if a is not None and v is not None]),
anchor)
2020-01-23 17:48:26 +00:00
2020-09-23 20:45:39 +00:00
def tableHeader(cell_values, cell_attributes = []):
return tr(
''.join([th(v, a)
for v, a in itertools.zip_longest(
cell_values, cell_attributes,
fillvalue = '')
if a is not None and v is not None]))
2020-01-23 17:48:26 +00:00
2020-02-26 16:18:58 +00:00
def tableStart(title):
cls = '-'.join(title.lower().split(' ')[:3]);
2020-08-05 15:46:05 +00:00
global table_anchor
table_anchor = cls
anchor = currentTableAnchor()
help_anchor = '-'.join(title.lower().split(' '));
return f"""
<h2 id="{anchor}">
<a class="cancela" href="#{anchor}">{title}</a>
<a class="cancela" href="https://github.com/ClickHouse/ClickHouse/tree/master/docker/test/performance-comparison#{help_anchor}"><sup style="color: #888">?</sup></a>
</h2>
<table class="{cls}">
"""
2020-02-26 16:18:58 +00:00
def tableEnd():
return '</table>'
2020-02-27 17:57:08 +00:00
def tsvRows(n):
2020-02-26 16:18:58 +00:00
try:
with open(n, encoding='utf-8') as fd:
result = []
for row in csv.reader(fd, delimiter="\t", quoting=csv.QUOTE_NONE):
new_row = []
for e in row:
2021-11-25 12:37:00 +00:00
# The first one .encode('utf-8').decode('unicode-escape') decodes the escape characters from the strings.
# The second one (encode('latin1').decode('utf-8')) fixes the changes with unicode vs utf-8 chars, so
# 'Чем зÐ<C2B7>нимаеÑ<C2B5>ЬÑ<C2AC>Ñ<EFBFBD>' is transformed back into 'Чем зАнимаешЬся'.
new_row.append(e.encode('utf-8').decode('unicode-escape').encode('latin1').decode('utf-8'))
result.append(new_row)
return result
2020-02-26 16:18:58 +00:00
except:
report_errors.append(
traceback.format_exception_only(
*sys.exc_info()[:2])[-1])
pass
return []
2020-02-27 17:57:08 +00:00
def htmlRows(n):
rawRows = tsvRows(n)
2020-01-23 17:48:26 +00:00
result = ''
2020-02-26 16:18:58 +00:00
for row in rawRows:
result += tableRow(row)
2020-01-23 17:48:26 +00:00
return result
2020-08-05 15:46:05 +00:00
def addSimpleTable(caption, columns, rows, pos=None):
global tables
text = ''
2020-02-27 17:57:08 +00:00
if not rows:
return
2020-08-05 15:46:05 +00:00
text += tableStart(caption)
text += tableHeader(columns)
2020-02-27 17:57:08 +00:00
for row in rows:
2020-08-05 15:46:05 +00:00
text += tableRow(row)
text += tableEnd()
tables.insert(pos if pos else len(tables), text)
2020-02-27 17:57:08 +00:00
2020-08-05 15:46:05 +00:00
def add_tested_commits():
2020-04-30 08:36:33 +00:00
global report_errors
try:
addSimpleTable('Tested Commits', ['Old', 'New'],
2020-04-30 08:36:33 +00:00
[['<pre>{}</pre>'.format(x) for x in
[open('left-commit.txt').read(),
open('right-commit.txt').read()]]])
except:
# Don't fail if no commit info -- maybe it's a manual run.
report_errors.append(
traceback.format_exception_only(
*sys.exc_info()[:2])[-1])
pass
2020-08-05 15:46:05 +00:00
def add_report_errors():
global tables
2020-04-30 08:36:33 +00:00
global report_errors
# Add the errors reported by various steps of comparison script
try:
report_errors += [l.strip() for l in open('report/errors.log')]
except:
report_errors.append(
traceback.format_exception_only(
*sys.exc_info()[:2])[-1])
pass
2020-08-05 15:46:05 +00:00
if not report_errors:
return
text = tableStart('Errors while Building the Report')
2020-08-05 15:46:05 +00:00
text += tableHeader(['Error'])
for x in report_errors:
text += tableRow([x])
text += tableEnd()
# Insert after Tested Commits
tables.insert(1, text)
2020-08-05 16:07:37 +00:00
errors_explained.append([f'<a href="#{currentTableAnchor()}">There were some errors while building the report</a>']);
2020-08-05 15:46:05 +00:00
def add_errors_explained():
2020-08-06 13:28:51 +00:00
if not errors_explained:
return
2020-08-07 01:25:45 +00:00
text = '<a name="fail1"/>'
text += tableStart('Error Summary')
2020-08-06 13:28:51 +00:00
text += tableHeader(['Description'])
for row in errors_explained:
text += tableRow(row)
text += tableEnd()
2020-08-05 15:46:05 +00:00
global tables
2020-08-06 13:28:51 +00:00
tables.insert(1, text)
2020-08-05 15:46:05 +00:00
2020-04-30 08:36:33 +00:00
2020-04-02 18:44:58 +00:00
if args.report == 'main':
2020-10-02 16:54:07 +00:00
print((header_template.format()))
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_tested_commits()
2020-04-02 18:44:58 +00:00
2020-05-22 02:51:26 +00:00
run_error_rows = tsvRows('run-errors.tsv')
error_tests += len(run_error_rows)
addSimpleTable('Run Errors', ['Test', 'Error'], run_error_rows)
2020-08-05 15:46:05 +00:00
if run_error_rows:
2020-08-05 16:07:37 +00:00
errors_explained.append([f'<a href="#{currentTableAnchor()}">There were some errors while running the tests</a>']);
2020-08-05 15:46:05 +00:00
2020-05-22 02:51:26 +00:00
slow_on_client_rows = tsvRows('report/slow-on-client.tsv')
error_tests += len(slow_on_client_rows)
addSimpleTable('Slow on Client',
2020-06-25 20:19:27 +00:00
['Client time,&nbsp;s', 'Server time,&nbsp;s', 'Ratio', 'Test', 'Query'],
2020-05-22 02:51:26 +00:00
slow_on_client_rows)
2020-08-05 15:46:05 +00:00
if slow_on_client_rows:
2020-08-05 16:07:37 +00:00
errors_explained.append([f'<a href="#{currentTableAnchor()}">Some queries are taking noticeable time client-side (missing `FORMAT Null`?)</a>']);
2020-06-23 12:09:54 +00:00
unmarked_short_rows = tsvRows('report/unexpected-query-duration.tsv')
2020-06-23 12:09:54 +00:00
error_tests += len(unmarked_short_rows)
addSimpleTable('Unexpected Query Duration',
['Problem', 'Marked as "short"?', 'Run time, s', 'Test', '#', 'Query'],
2020-06-23 12:09:54 +00:00
unmarked_short_rows)
2020-08-05 15:46:05 +00:00
if unmarked_short_rows:
errors_explained.append([f'<a href="#{currentTableAnchor()}">Some queries have unexpected duration</a>']);
2020-05-22 02:51:26 +00:00
2020-08-05 15:46:05 +00:00
def add_partial():
rows = tsvRows('report/partial-queries-report.tsv')
if not rows:
return
2020-08-05 15:46:05 +00:00
global unstable_partial_queries, slow_average_tests, tables
text = tableStart('Partial Queries')
columns = ['Median time, s', 'Relative time variance', 'Test', '#', 'Query']
2020-08-05 15:46:05 +00:00
text += tableHeader(columns)
attrs = ['' for c in columns]
for row in rows:
2020-08-07 01:25:45 +00:00
anchor = f'{currentTableAnchor()}.{row[2]}.{row[3]}'
if float(row[1]) > 0.10:
attrs[1] = f'style="background: {color_bad}"'
unstable_partial_queries += 1
2020-08-07 01:25:45 +00:00
errors_explained.append([f'<a href="#{anchor}">The query no. {row[3]} of test \'{row[2]}\' has excessive variance of run time. Keep it below 10%</a>'])
else:
attrs[1] = ''
if float(row[0]) > allowed_single_run_time:
attrs[0] = f'style="background: {color_bad}"'
2020-08-12 08:54:46 +00:00
errors_explained.append([f'<a href="#{anchor}">The query no. {row[3]} of test \'{row[2]}\' is taking too long to run. Keep the run time below {allowed_single_run_time} seconds"</a>'])
slow_average_tests += 1
else:
attrs[0] = ''
2020-08-07 01:25:45 +00:00
text += tableRow(row, attrs, anchor)
2020-08-05 15:46:05 +00:00
text += tableEnd()
tables.append(text)
2020-08-05 15:46:05 +00:00
add_partial()
2020-08-05 15:46:05 +00:00
def add_changes():
2020-04-28 07:45:35 +00:00
rows = tsvRows('report/changed-perf.tsv')
2020-04-02 18:44:58 +00:00
if not rows:
return
2020-08-05 15:46:05 +00:00
global faster_queries, slower_queries, tables
2020-04-02 18:44:58 +00:00
text = tableStart('Changes in Performance')
2020-04-02 18:44:58 +00:00
columns = [
2020-06-25 20:19:27 +00:00
'Old,&nbsp;s', # 0
'New,&nbsp;s', # 1
2020-09-03 00:57:25 +00:00
'Ratio of speedup&nbsp;(-) or slowdown&nbsp;(+)', # 2
'Relative difference (new&nbsp;&minus;&nbsp;old) / old', # 3
2020-09-10 15:48:39 +00:00
'p&nbsp;<&nbsp;0.01 threshold', # 4
2020-09-23 20:45:39 +00:00
'', # Failed # 5
'Test', # 6
'#', # 7
'Query', # 8
2020-04-02 18:44:58 +00:00
]
attrs = ['' for c in columns]
2020-08-01 10:10:36 +00:00
attrs[5] = None
2020-09-23 20:45:39 +00:00
text += tableHeader(columns, attrs)
2020-04-02 18:44:58 +00:00
for row in rows:
2020-08-07 01:25:45 +00:00
anchor = f'{currentTableAnchor()}.{row[6]}.{row[7]}'
if int(row[5]):
if float(row[3]) < 0.:
2020-04-21 18:46:45 +00:00
faster_queries += 1
attrs[2] = attrs[3] = f'style="background: {color_good}"'
2020-04-21 18:46:45 +00:00
else:
slower_queries += 1
attrs[2] = attrs[3] = f'style="background: {color_bad}"'
2020-08-07 01:25:45 +00:00
errors_explained.append([f'<a href="#{anchor}">The query no. {row[7]} of test \'{row[6]}\' has slowed down</a>'])
2020-04-28 07:45:35 +00:00
else:
attrs[2] = attrs[3] = ''
2020-04-02 18:44:58 +00:00
2020-08-07 01:25:45 +00:00
text += tableRow(row, attrs, anchor)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text += tableEnd()
tables.append(text)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_changes()
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
def add_unstable_queries():
global unstable_queries, very_unstable_queries, tables
2020-04-02 18:44:58 +00:00
2020-04-28 07:45:35 +00:00
unstable_rows = tsvRows('report/unstable-queries.tsv')
2020-04-02 18:44:58 +00:00
if not unstable_rows:
return
unstable_queries += len(unstable_rows)
columns = [
2020-06-25 20:19:27 +00:00
'Old,&nbsp;s', #0
'New,&nbsp;s', #1
2020-04-02 18:44:58 +00:00
'Relative difference (new&nbsp;-&nbsp;old)/old', #2
2020-09-10 15:48:39 +00:00
'p&nbsp;&lt;&nbsp;0.01 threshold', #3
2020-09-23 20:45:39 +00:00
'', # Failed #4
2020-04-28 07:45:35 +00:00
'Test', #5
2020-06-08 13:57:33 +00:00
'#', #6
'Query' #7
2020-02-27 17:57:08 +00:00
]
2020-09-23 20:45:39 +00:00
attrs = ['' for c in columns]
attrs[4] = None
2020-02-27 17:57:08 +00:00
text = tableStart('Unstable Queries')
2020-09-23 20:45:39 +00:00
text += tableHeader(columns, attrs)
2020-04-02 18:44:58 +00:00
for r in unstable_rows:
2020-08-07 01:25:45 +00:00
anchor = f'{currentTableAnchor()}.{r[5]}.{r[6]}'
2020-04-28 07:45:35 +00:00
if int(r[4]):
2020-04-02 18:44:58 +00:00
very_unstable_queries += 1
attrs[3] = f'style="background: {color_bad}"'
2020-04-02 18:44:58 +00:00
else:
attrs[3] = ''
2021-05-13 09:16:43 +00:00
# Just don't add the slightly unstable queries we don't consider
# errors. It's not clear what the user should do with them.
continue
2020-04-02 18:44:58 +00:00
2020-08-07 01:25:45 +00:00
text += tableRow(r, attrs, anchor)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text += tableEnd()
# Don't add an empty table.
if very_unstable_queries:
tables.append(text)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_unstable_queries()
2020-04-02 18:44:58 +00:00
skipped_tests_rows = tsvRows('analyze/skipped-tests.tsv')
addSimpleTable('Skipped Tests', ['Test', 'Reason'], skipped_tests_rows)
2020-04-02 18:44:58 +00:00
addSimpleTable('Test Performance Changes',
2020-09-03 00:57:25 +00:00
['Test', 'Ratio of speedup&nbsp;(-) or slowdown&nbsp;(+)', 'Queries', 'Total not OK', 'Changed perf', 'Unstable'],
2020-06-09 13:29:07 +00:00
tsvRows('report/test-perf-changes.tsv'))
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
def add_test_times():
global slow_average_tests, tables
2020-04-28 07:45:35 +00:00
rows = tsvRows('report/test-times.tsv')
2020-04-02 18:44:58 +00:00
if not rows:
return
columns = [
2020-10-20 17:06:51 +00:00
'Test', #0
'Wall clock time, entire test,&nbsp;s', #1
'Total client time for measured query runs,&nbsp;s', #2
'Queries', #3
'Longest query, total for measured runs,&nbsp;s', #4
'Wall clock time per query,&nbsp;s', #5
'Shortest query, total for measured runs,&nbsp;s', #6
'', # Runs #7
2020-04-02 18:44:58 +00:00
]
2020-09-23 07:28:28 +00:00
attrs = ['' for c in columns]
attrs[7] = None
2020-04-02 18:44:58 +00:00
text = tableStart('Test Times')
2020-09-23 20:45:39 +00:00
text += tableHeader(columns, attrs)
2020-05-09 15:58:47 +00:00
allowed_average_run_time = 3.75 # 60 seconds per test at (7 + 1) * 2 runs
2020-04-02 18:44:58 +00:00
for r in rows:
2020-09-01 19:05:57 +00:00
anchor = f'{currentTableAnchor()}.{r[0]}'
total_runs = (int(r[7]) + 1) * 2 # one prewarm run, two servers
2020-09-30 12:03:52 +00:00
if r[0] != 'Total' and float(r[5]) > allowed_average_run_time * total_runs:
2020-04-02 18:44:58 +00:00
# FIXME should be 15s max -- investigate parallel_insert
slow_average_tests += 1
2020-09-03 01:42:25 +00:00
attrs[5] = f'style="background: {color_bad}"'
2020-09-01 19:05:57 +00:00
errors_explained.append([f'<a href="#{anchor}">The test \'{r[0]}\' is too slow to run as a whole. Investigate whether the create and fill queries can be sped up'])
2020-04-02 18:44:58 +00:00
else:
2020-09-03 01:42:25 +00:00
attrs[5] = ''
2020-04-02 18:44:58 +00:00
2020-09-30 12:03:52 +00:00
if r[0] != 'Total' and float(r[4]) > allowed_single_run_time * total_runs:
2020-04-02 18:44:58 +00:00
slow_average_tests += 1
2020-09-03 01:42:25 +00:00
attrs[4] = f'style="background: {color_bad}"'
2020-08-07 01:25:45 +00:00
errors_explained.append([f'<a href="./all-queries.html#all-query-times.{r[0]}.0">Some query of the test \'{r[0]}\' is too slow to run. See the all queries report'])
2020-04-02 18:44:58 +00:00
else:
2020-09-03 01:42:25 +00:00
attrs[4] = ''
2020-04-02 18:44:58 +00:00
2020-09-01 19:05:57 +00:00
text += tableRow(r, attrs, anchor)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text += tableEnd()
tables.append(text)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_test_times()
2020-04-02 18:44:58 +00:00
addSimpleTable('Metric Changes',
2020-06-25 20:19:27 +00:00
['Metric', 'Old median value', 'New median value',
'Relative difference', 'Times difference'],
tsvRows('metrics/changes.tsv'))
2020-08-05 15:46:05 +00:00
add_report_errors()
add_errors_explained()
for t in tables:
print(t)
2020-04-02 18:44:58 +00:00
2021-04-23 13:47:13 +00:00
print(f"""
2020-08-28 22:46:04 +00:00
</div>
2020-04-02 18:44:58 +00:00
<p class="links">
<a href="all-queries.html">All queries</a>
<a href="compare.log">Log</a>
<a href="output.7z">Test output</a>
2021-04-23 13:47:13 +00:00
{os.getenv("CHPC_ADD_REPORT_LINKS") or ''}
2020-04-02 18:44:58 +00:00
</p>
</body>
</html>
""")
status = 'success'
message = 'See the report'
message_array = []
if slow_average_tests:
status = 'failure'
message_array.append(str(slow_average_tests) + ' too long')
if faster_queries:
message_array.append(str(faster_queries) + ' faster')
if slower_queries:
if slower_queries > 3:
status = 'failure'
2020-04-02 18:44:58 +00:00
message_array.append(str(slower_queries) + ' slower')
if unstable_partial_queries:
very_unstable_queries += unstable_partial_queries
status = 'failure'
2021-05-12 22:32:53 +00:00
# Don't show mildly unstable queries, only the very unstable ones we
# treat as errors.
if very_unstable_queries:
2021-06-27 00:14:42 +00:00
if very_unstable_queries > 5:
2021-06-04 15:27:21 +00:00
error_tests += very_unstable_queries
status = 'failure'
2021-05-20 12:05:20 +00:00
message_array.append(str(very_unstable_queries) + ' unstable')
2020-04-02 18:44:58 +00:00
error_tests += slow_average_tests
if error_tests:
status = 'failure'
2020-06-10 21:38:26 +00:00
message_array.insert(0, str(error_tests) + ' errors')
2020-04-02 18:44:58 +00:00
if message_array:
message = ', '.join(message_array)
if report_errors:
status = 'failure'
message = 'Errors while building the report.'
2020-10-02 16:54:07 +00:00
print(("""
2020-04-02 18:44:58 +00:00
<!--status: {status}-->
<!--message: {message}-->
2020-10-02 16:54:07 +00:00
""".format(status=status, message=message)))
2020-04-02 18:44:58 +00:00
elif args.report == 'all-queries':
2020-10-02 16:54:07 +00:00
print((header_template.format()))
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_tested_commits()
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
def add_all_queries():
2020-04-28 07:45:35 +00:00
rows = tsvRows('report/all-queries.tsv')
2020-04-02 18:44:58 +00:00
if not rows:
return
columns = [
2020-09-23 20:45:39 +00:00
'', # Changed #0
'', # Unstable #1
2020-06-25 20:19:27 +00:00
'Old,&nbsp;s', #2
'New,&nbsp;s', #3
2020-09-03 00:57:25 +00:00
'Ratio of speedup&nbsp;(-) or slowdown&nbsp;(+)', #4
'Relative difference (new&nbsp;&minus;&nbsp;old) / old', #5
2020-09-10 15:48:39 +00:00
'p&nbsp;&lt;&nbsp;0.01 threshold', #6
2020-04-28 07:45:35 +00:00
'Test', #7
2020-06-08 13:57:33 +00:00
'#', #8
'Query', #9
2020-04-02 18:44:58 +00:00
]
attrs = ['' for c in columns]
2020-04-28 07:45:35 +00:00
attrs[0] = None
attrs[1] = None
2020-09-23 20:45:39 +00:00
text = tableStart('All Query Times')
text += tableHeader(columns, attrs)
2020-04-02 18:44:58 +00:00
for r in rows:
2020-08-07 01:25:45 +00:00
anchor = f'{currentTableAnchor()}.{r[7]}.{r[8]}'
2020-04-28 07:45:35 +00:00
if int(r[1]):
attrs[6] = f'style="background: {color_bad}"'
2020-04-21 18:46:45 +00:00
else:
2020-04-28 07:45:35 +00:00
attrs[6] = ''
2020-04-21 18:46:45 +00:00
2020-04-28 07:45:35 +00:00
if int(r[0]):
if float(r[5]) > 0.:
attrs[4] = attrs[5] = f'style="background: {color_bad}"'
2020-04-21 18:46:45 +00:00
else:
attrs[4] = attrs[5] = f'style="background: {color_good}"'
2020-04-02 18:44:58 +00:00
else:
attrs[4] = attrs[5] = ''
2020-04-02 18:44:58 +00:00
2020-04-29 17:26:28 +00:00
if (float(r[2]) + float(r[3])) / 2 > allowed_single_run_time:
attrs[2] = f'style="background: {color_bad}"'
attrs[3] = f'style="background: {color_bad}"'
2020-04-29 17:26:28 +00:00
else:
attrs[2] = ''
attrs[3] = ''
2020-08-07 01:25:45 +00:00
text += tableRow(r, attrs, anchor)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text += tableEnd()
tables.append(text)
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_all_queries()
add_report_errors()
for t in tables:
print(t)
2020-04-30 08:36:33 +00:00
2021-04-23 13:47:13 +00:00
print(f"""
2020-08-28 22:46:04 +00:00
</div>
2020-04-02 18:44:58 +00:00
<p class="links">
<a href="report.html">Main report</a>
<a href="compare.log">Log</a>
<a href="output.7z">Test output</a>
2021-04-23 13:47:13 +00:00
{os.getenv("CHPC_ADD_REPORT_LINKS") or ''}
2020-04-02 18:44:58 +00:00
</p>
</body>
</html>
""")