mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Performance comparison fixes
This commit is contained in:
Alexander Kuzmenkov
parent
4075f26583
commit
cd88b5380c
@ -97,10 +97,6 @@ function run_tests
|
||||
touch "$x"
|
||||
done
|
||||
|
||||
# FIXME remove some broken long tests
|
||||
rm "$test_prefix"/{IPv4,IPv6,modulo,parse_engine_file,number_formatting_formats,select_format}.xml ||:
|
||||
|
||||
test_files=$(ls "$test_prefix"/*.xml)
|
||||
|
||||
# FIXME a quick crutch to bring the run time down for the unstable tests --
|
||||
# if some performance tests xmls were changed in a PR, run only these ones.
|
||||
@ -126,6 +122,17 @@ function run_tests
|
||||
test_files=$(ls "$test_prefix"/$CHPC_TEST_GLOB.xml)
|
||||
fi
|
||||
|
||||
if [ "$test_files" == "" ]
|
||||
then
|
||||
# FIXME remove some broken long tests
|
||||
for test_name in {IPv4,IPv6,modulo,parse_engine_file,number_formatting_formats,select_format,arithmetic,cryptographic_hashes,logical_functions_{medium,small}}
|
||||
do
|
||||
printf "$test_name\tMarked as broken (see compare.sh)" >> skipped-tests.tsv
|
||||
rm "$test_prefix/$test_name.xml" ||:
|
||||
done
|
||||
test_files=$(ls "$test_prefix"/*.xml)
|
||||
fi
|
||||
|
||||
# Run the tests.
|
||||
test_name="<none>"
|
||||
for test in $test_files
|
||||
@ -275,9 +282,11 @@ create table test_times_tsv engine File(TSV, 'test-times.tsv') as
|
||||
from test_time join wall_clock using test
|
||||
order by avg_real_per_query desc;
|
||||
|
||||
create table all_queries_tsv engine File(TSV, 'all-queries.tsv') as
|
||||
select left, right, diff, rd, test, query
|
||||
from queries order by rd[3] desc;
|
||||
create table all_tests_tsv engine File(TSV, 'all-queries.tsv') as
|
||||
select left, right, diff,
|
||||
floor(left > right ? left / right : right / left, 3),
|
||||
rd, test, query
|
||||
from queries order by test, query;
|
||||
" 2> >(head -2 >> report-errors.rep) ||:
|
||||
|
||||
for version in {right,left}
|
||||
@ -429,6 +438,7 @@ case "$stage" in
|
||||
"report")
|
||||
time report ||:
|
||||
|
||||
time "$script_dir/report.py" --report=all-queries > all-queries.html 2> >(head -2 >> report-errors.rep) ||:
|
||||
time "$script_dir/report.py" > report.html
|
||||
;&
|
||||
esac
|
||||
|
||||
@ -90,17 +90,23 @@ export PYTHONIOENCODING=utf-8
|
||||
# Use a default number of runs if not told otherwise
|
||||
export CHPC_RUNS=${CHPC_RUNS:-7}
|
||||
|
||||
# By default, use the main comparison script from the tested package, so that we
|
||||
# can change it in PRs.
|
||||
script_path="right/scripts"
|
||||
if [ -v CHPC_LOCAL_SCRIPT ]
|
||||
then
|
||||
script_path=".."
|
||||
fi
|
||||
|
||||
# Even if we have some errors, try our best to save the logs.
|
||||
set +e
|
||||
|
||||
# Use main comparison script from the tested package, so that we can change it
|
||||
# in PRs.
|
||||
# Older version use 'kill 0', so put the script into a separate process group
|
||||
# FIXME remove set +m in April 2020
|
||||
set +m
|
||||
{ \
|
||||
time ../download.sh "$REF_PR" "$REF_SHA" "$PR_TO_TEST" "$SHA_TO_TEST" && \
|
||||
time stage=configure right/scripts/compare.sh ; \
|
||||
time stage=configure "$script_path"/compare.sh ; \
|
||||
} 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee compare.log
|
||||
set -m
|
||||
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import collections
|
||||
import csv
|
||||
@ -8,6 +9,11 @@ import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
parser = argparse.ArgumentParser(description='Create performance test report')
|
||||
parser.add_argument('--report', default='main', choices=['main', 'all-queries'],
|
||||
help='Which report to build')
|
||||
args = parser.parse_args()
|
||||
|
||||
report_errors = []
|
||||
error_tests = 0
|
||||
slow_average_tests = 0
|
||||
@ -16,7 +22,7 @@ slower_queries = 0
|
||||
unstable_queries = 0
|
||||
very_unstable_queries = 0
|
||||
|
||||
print("""
|
||||
header_template = """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<style>
|
||||
@ -56,7 +62,7 @@ tr:nth-child(odd) td {{filter: brightness(95%);}}
|
||||
<div class="main">
|
||||
|
||||
<h1>ClickHouse performance comparison</h1>
|
||||
""".format())
|
||||
"""
|
||||
|
||||
table_anchor = 0
|
||||
row_anchor = 0
|
||||
@ -133,6 +139,9 @@ def printSimpleTable(caption, columns, rows):
|
||||
print(tableRow(row))
|
||||
print(tableEnd())
|
||||
|
||||
if args.report == 'main':
|
||||
print(header_template.format())
|
||||
|
||||
printSimpleTable('Tested commits', ['Old', 'New'],
|
||||
[['<pre>{}</pre>'.format(x) for x in
|
||||
[open('left-commit.txt').read(),
|
||||
@ -282,6 +291,7 @@ if len(report_errors):
|
||||
print("""
|
||||
<p class="links">
|
||||
<a href="output.7z">Test output</a>
|
||||
<a href="all-queries.html">All queries</a>
|
||||
<a href="compare.log">Log</a>
|
||||
</p>
|
||||
</body>
|
||||
@ -325,3 +335,56 @@ print("""
|
||||
<!--status: {status}-->
|
||||
<!--message: {message}-->
|
||||
""".format(status=status, message=message))
|
||||
|
||||
elif args.report == 'all-queries':
|
||||
|
||||
print(header_template.format())
|
||||
|
||||
printSimpleTable('Tested commits', ['Old', 'New'],
|
||||
[['<pre>{}</pre>'.format(x) for x in
|
||||
[open('left-commit.txt').read(),
|
||||
open('right-commit.txt').read()]]])
|
||||
|
||||
def print_all_queries():
|
||||
rows = tsvRows('all-queries.tsv')
|
||||
if not rows:
|
||||
return
|
||||
|
||||
columns = [
|
||||
'Old, s', #0
|
||||
'New, s', #1
|
||||
'Relative difference (new - old)/old', #2
|
||||
'Times speedup/slowdown', #3
|
||||
'Randomization distribution quantiles \
|
||||
[5%, 50%, 95%, 99%]', #4
|
||||
'Test', #5
|
||||
'Query', #6
|
||||
]
|
||||
|
||||
print(tableStart('All query times'))
|
||||
print(tableHeader(columns))
|
||||
|
||||
attrs = ['' for c in columns]
|
||||
for r in rows:
|
||||
if float(r[2]) > 0.05:
|
||||
attrs[3] = 'style="background: #ffb0a0"'
|
||||
elif float(r[2]) < -0.05:
|
||||
attrs[3] = 'style="background: #adbdff"'
|
||||
else:
|
||||
attrs[3] = ''
|
||||
|
||||
print(tableRow(r, attrs))
|
||||
|
||||
print(tableEnd())
|
||||
|
||||
print_all_queries()
|
||||
|
||||
print("""
|
||||
<p class="links">
|
||||
<a href="output.7z">Test output</a>
|
||||
<a href="report.html">Main report</a>
|
||||
<a href="compare.log">Log</a>
|
||||
</p>
|
||||
</body>
|
||||
</html>
|
||||
""")
|
||||
|
||||
File diff suppressed because one or more lines are too long
@ -1,16 +1,8 @@
|
||||
<test>
|
||||
|
||||
<stop_conditions>
|
||||
<all_of>
|
||||
<total_time_ms>10000</total_time_ms>
|
||||
</all_of>
|
||||
</stop_conditions>
|
||||
|
||||
|
||||
<query>SELECT arraySlice(arrayFill(x -> ((x % 2) >= 0), range(100000000)), 1, 10)</query>
|
||||
<query>SELECT arraySlice(arrayFill(x -> (((x.1) % 2) >= 0), arrayMap(x -> (x, toString(x)), range(100000000))), 1, 10)</query>
|
||||
<query>SELECT arraySlice(arrayFill(x -> ((x % 2) >= 2), range(100000000)), 1, 10)</query>
|
||||
<query>SELECT arraySlice(arrayFill(x -> (((x.1) % 2) >= 2), arrayMap(x -> (x, toString(x)), range(100000000))), 1, 10)</query>
|
||||
<query>SELECT arraySlice(arrayFill(x -> ((x % 2) = 0), range(100000000)), 1, 10)</query>
|
||||
<query>SELECT arraySlice(arrayFill(x -> (((x.1) % 2) = 0), arrayMap(x -> (x, toString(x)), range(100000000))), 1, 10)</query>
|
||||
<query>SELECT arraySlice(arrayFill(x -> ((x % 2) >= 0), range(100000000)), 1, 10) FORMAT Null</query>
|
||||
<query>SELECT arraySlice(arrayFill(x -> (((x.1) % 2) >= 0), arrayMap(x -> (x, toString(x)), range(100000000))), 1, 10) FORMAT Null</query>
|
||||
<query>SELECT arraySlice(arrayFill(x -> ((x % 2) >= 2), range(100000000)), 1, 10) FORMAT Null</query>
|
||||
<query>SELECT arraySlice(arrayFill(x -> (((x.1) % 2) >= 2), arrayMap(x -> (x, toString(x)), range(100000000))), 1, 10) FORMAT Null</query>
|
||||
<query>SELECT arraySlice(arrayFill(x -> ((x % 2) = 0), range(100000000)), 1, 10) FORMAT Null</query>
|
||||
<query>SELECT arraySlice(arrayFill(x -> (((x.1) % 2) = 0), arrayMap(x -> (x, toString(x)), range(100000000))), 1, 10) FORMAT Null</query>
|
||||
</test>
|
||||
|
||||
@ -1,17 +1,4 @@
|
||||
<test>
|
||||
|
||||
<stop_conditions>
|
||||
<all_of>
|
||||
<iterations>5</iterations>
|
||||
<min_time_not_changing_for_ms>10000</min_time_not_changing_for_ms>
|
||||
</all_of>
|
||||
<any_of>
|
||||
<iterations>50</iterations>
|
||||
<total_time_ms>60000</total_time_ms>
|
||||
</any_of>
|
||||
</stop_conditions>
|
||||
|
||||
|
||||
<preconditions>
|
||||
<table_exists>test.hits</table_exists>
|
||||
</preconditions>
|
||||
@ -24,13 +11,13 @@
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(concat(MobilePhoneModel, 'Hello'))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(concat(PageCharset, 'a'))</query>
|
||||
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{}{}', URL, URL))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{}{}', URL, SearchPhrase))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{}{}', MobilePhoneModel, SearchPhrase))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{}Hello', URL))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('World{}', SearchPhrase))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{}Hello', MobilePhoneModel))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{}a', PageCharset))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{{}}{{}}', URL, URL))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{{}}{{}}', URL, SearchPhrase))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{{}}{{}}', MobilePhoneModel, SearchPhrase))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{{}}Hello', URL))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('World{{}}', SearchPhrase))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{{}}Hello', MobilePhoneModel))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{{}}a', PageCharset))</query>
|
||||
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(concat(URL, URL, URL))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(concat(URL, SearchPhrase, MobilePhoneModel))</query>
|
||||
@ -39,10 +26,10 @@
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(concat(MobilePhoneModel, 'Hello', PageCharset))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(concat('a', PageCharset, 'b'))</query>
|
||||
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{}{}{}', URL, URL, URL))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{}{}{}', URL, SearchPhrase, MobilePhoneModel))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{}Hello{}', URL, URL))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('Hello{}World', SearchPhrase))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{}Hello{}', MobilePhoneModel, PageCharset))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('a{}b', PageCharset))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{{}}{{}}{{}}', URL, URL, URL))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{{}}{{}}{{}}', URL, SearchPhrase, MobilePhoneModel))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{{}}Hello{{}}', URL, URL))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('Hello{{}}World', SearchPhrase))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('{{}}Hello{{}}', MobilePhoneModel, PageCharset))</query>
|
||||
<query>SELECT count() FROM test.hits WHERE NOT ignore(format('a{{}}b', PageCharset))</query>
|
||||
</test>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user