thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

[wip] performance comparison

Browse Source
This commit is contained in:
Alexander Kuzmenkov 2020-01-23 20:48:26 +03:00
parent f563d7f4b3
commit c7616ff11a
4 changed files with 115 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docker/test/performance-comparison/compare.sh
View File

@ -146,7 +146,9 @@ run_tests
# Analyze results
result_structure="left float, right float, diff float, rd Array(float), query text"
right/clickhouse local --file '*-report.tsv' -S "$result_structure" --query "select * from table where diff < 0.05 and rd[3] > 0.05 order by rd[3] desc" > flap-prone.tsv
right/clickhouse local --file '*-report.tsv' -S "$result_structure" --query "select * from table where diff > 0.05 and diff > rd[3] order by diff desc" > bad-perf.tsv
right/clickhouse local --file '*-client-time.tsv' -S "query text, client float, server float" -q "select *, floor(client/server, 3) p from table order by p desc" > client-time.tsv
right/clickhouse local --file '*-report.tsv' -S "$result_structure" --query "select * from table where abs(diff) < 0.05 and rd[3] > 0.05 order by rd[3] desc" > unstable.tsv
right/clickhouse local --file '*-report.tsv' -S "$result_structure" --query "select * from table where abs(diff) > 0.05 and abs(diff) > rd[3] order by diff desc" > changed-perf.tsv
right/clickhouse local --file '*-client-time.tsv' -S "query text, client float, server float" -q "select client, server, floor(client/server, 3) p, query from table where p > 1.01 order by p desc" > slow-on-client.tsv
grep Exception:[^:] *-err.log > run-errors.log
./report.py > report.html

2
docker/test/performance-comparison/entrypoint.sh
View File

@ -29,5 +29,5 @@ set -m
time ../compare.sh 0 $ref_sha $PR_TO_TEST $SHA_TO_TEST 2>&1 | ts | tee compare.log
set +m
7z a /output/output.7z *.log *.tsv
7z a /output/output.7z *.log *.tsv *.html
cp compare.log /output

8
docker/test/performance-comparison/eqmed.sql
View File

@ -1,10 +1,10 @@
-- input is table(query text, run UInt32, version int, time float)
select
-- abs(diff_percent) > rd_quantiles_percent[3] fail,
floor(original_medians_array.time_by_version[1], 4) m1,
floor(original_medians_array.time_by_version[2], 4) m2,
floor((m1 - m2) / m1, 3) diff_percent,
arrayMap(x -> floor(x / m1, 3), rd.rd_quantiles) rd_quantiles_percent,
floor(original_medians_array.time_by_version[1], 4) left,
floor(original_medians_array.time_by_version[2], 4) right,
floor((right - left) / left, 3) diff_percent,
arrayMap(x -> floor(x / left, 3), rd.rd_quantiles) rd_quantiles_percent,
query
from
(

105
docker/test/performance-comparison/report.py Executable file
View File

@ -0,0 +1,105 @@
#!/usr/bin/python3
import collections
import csv
import os
import sys
doc_template = """
<!DOCTYPE html>
<html>
<style>
@font-face {{
font-family:'Yandex Sans Display Web';
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot);
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot?#iefix) format('embedded-opentype'),
url(https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2) format('woff2'),
url(https://yastatic.net/adv-www/_/v2Sve_obH3rKm6rKrtSQpf-eB7U.woff) format('woff'),
url(https://yastatic.net/adv-www/_/PzD8hWLMunow5i3RfJ6WQJAL7aI.ttf) format('truetype'),
url(https://yastatic.net/adv-www/_/lF_KG5g4tpQNlYIgA0e77fBSZ5s.svg#YandexSansDisplayWeb-Regular) format('svg');
font-weight:400;
font-style:normal;
font-stretch:normal
}}
body {{ font-family: "Yandex Sans Display Web", Arial, sans-serif; background: #EEE; }}
h1 {{ margin-left: 10px; }}
th, td {{ border: 0; padding: 5px 10px 5px 10px; text-align: left; vertical-align: top; line-height: 1.5; background-color: #FFF;
td {{ white-space: pre; font-family: Monospace, Courier New; }}
border: 0; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }}
a {{ color: #06F; text-decoration: none; }}
a:hover, a:active {{ color: #F40; text-decoration: underline; }}
table {{ border: 0; }}
.main {{ margin-left: 10%; }}
p.links a {{ padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-space: nowrap; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }}
</style>
<title>{header}</title>
</head>
<body>
<div class="main">
<h1>{header}</h1>
{test_part}
<p class="links">
<a href="{raw_log_url}">{raw_log_name}</a>
<a href="{branch_url}">{branch_name}</a>
<a href="{commit_url}">Commit</a>
{additional_urls}
<a href="output.7z">Test output</a>
<a href="{task_url}">Task (private network)</a>
</p>
</body>
</html>
"""
table_template = """
<h2>{caption}</h2>
<table>
{header}
{rows}
</table>
"""
def tr(x):
return '<tr>' + str(x) + '</tr>'
def td(x):
return '<td>' + str(x) + '</td>'
def th(x):
return '<th>' + str(x) + '</th>'
def table_row(r):
return tr(''.join([td(f) for f in r]))
def table_header(r):
return tr(''.join([th(f) for f in r]))
def tsv_rows(n):
result = ''
with open(n) as fd:
for row in csv.reader(fd, delimiter="\t", quotechar='"'):
result += table_row(row)
return result
params = collections.defaultdict(str)
params['header'] = "ClickHouse Performance Comparison"
params['test_part'] = (table_template.format_map(
collections.defaultdict(str,
caption = 'Changes in performance',
header = table_header(['Left', 'Right', 'Diff', 'RD', 'Query']),
rows = tsv_rows('changed-perf.tsv'))) +
table_template.format(
caption = 'Slow on client',
header = table_header(['Client', 'Server', 'Ratio', 'Query']),
rows = tsv_rows('slow-on-client.tsv')) +
table_template.format(
caption = 'Unstable',
header = table_header(['Left', 'Right', 'Diff', 'RD', 'Query']),
rows = tsv_rows('unstable.tsv')) +
table_template.format(
caption = 'Run errors',
header = table_header(['A', 'B']),
rows = tsv_rows('run-errors.log'))
)
print(doc_template.format_map(params))