mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Remove concurrent benchmark from the perf test
It's not stable enough and has only secondary utility.
This commit is contained in:
parent
a6486490a2
commit
0db5b4a72c
@ -197,33 +197,9 @@ function run_tests
|
||||
wait
|
||||
}
|
||||
|
||||
# Run some queries concurrently and report the resulting TPS. This additional
|
||||
# (relatively) short test helps detect concurrency-related effects, because the
|
||||
# main performance comparison testing is done query-by-query.
|
||||
function run_benchmark
|
||||
{
|
||||
rm -rf benchmark ||:
|
||||
mkdir benchmark ||:
|
||||
|
||||
# The list is built by run_tests.
|
||||
while IFS= read -r file
|
||||
do
|
||||
name=$(basename "$file" ".xml")
|
||||
|
||||
"$script_dir/perf.py" --print-queries "$file" > "benchmark/$name-queries.txt"
|
||||
"$script_dir/perf.py" --print-settings "$file" > "benchmark/$name-settings.txt"
|
||||
|
||||
readarray -t settings < "benchmark/$name-settings.txt"
|
||||
command=(clickhouse-benchmark --concurrency 6 --cumulative --iterations 1000 --randomize 1 --delay 0 --continue_on_errors "${settings[@]}")
|
||||
|
||||
"${command[@]}" --port 9001 --json "benchmark/$name-left.json" < "benchmark/$name-queries.txt"
|
||||
"${command[@]}" --port 9002 --json "benchmark/$name-right.json" < "benchmark/$name-queries.txt"
|
||||
done < benchmarks-to-run.txt
|
||||
}
|
||||
|
||||
function get_profiles_watchdog
|
||||
{
|
||||
sleep 6000
|
||||
sleep 600
|
||||
|
||||
echo "The trace collection did not finish in time." >> profile-errors.log
|
||||
|
||||
@ -570,8 +546,8 @@ create table test_time_changes engine File(TSV, 'report/test-time-changes.tsv')
|
||||
select test, count(*) queries,
|
||||
sum(left) as left, sum(right) as right,
|
||||
(right - left) / right average_time_change
|
||||
from queries
|
||||
group by test
|
||||
from queries
|
||||
group by test
|
||||
order by abs(average_time_change) desc
|
||||
)
|
||||
;
|
||||
@ -980,9 +956,6 @@ case "$stage" in
|
||||
# Ignore the errors to collect the log and build at least some report, anyway
|
||||
time run_tests ||:
|
||||
;&
|
||||
"run_benchmark")
|
||||
time run_benchmark 2> >(tee -a run-errors.tsv 1>&2) ||:
|
||||
;&
|
||||
"get_profiles")
|
||||
# Check for huge pages.
|
||||
cat /sys/kernel/mm/transparent_hugepage/enabled > thp-enabled.txt ||:
|
||||
|
@ -497,73 +497,6 @@ if args.report == 'main':
|
||||
|
||||
add_test_times()
|
||||
|
||||
def add_benchmark_results():
|
||||
if not os.path.isfile('benchmark/website-left.json'):
|
||||
return
|
||||
|
||||
json_reports = [json.load(open(f'benchmark/website-{x}.json')) for x in ['left', 'right']]
|
||||
stats = [next(iter(x.values()))["statistics"] for x in json_reports]
|
||||
qps = [x["QPS"] for x in stats]
|
||||
queries = [x["num_queries"] for x in stats]
|
||||
errors = [x["num_errors"] for x in stats]
|
||||
relative_diff = (qps[1] - qps[0]) / max(0.01, qps[0]);
|
||||
times_diff = max(qps) / max(0.01, min(qps))
|
||||
|
||||
all_rows = []
|
||||
header = ['Benchmark', 'Metric', 'Old', 'New', 'Relative difference', 'Times difference'];
|
||||
|
||||
attrs = ['' for x in header]
|
||||
row = ['website', 'queries', f'{queries[0]:d}', f'{queries[1]:d}', '--', '--']
|
||||
attrs[0] = 'rowspan=2'
|
||||
all_rows.append([row, attrs])
|
||||
|
||||
attrs = ['' for x in header]
|
||||
row = [None, 'queries/s', f'{qps[0]:.3f}', f'{qps[1]:.3f}', f'{relative_diff:.3f}', f'x{times_diff:.3f}']
|
||||
if abs(relative_diff) > 0.1:
|
||||
# More queries per second is better.
|
||||
if relative_diff > 0.:
|
||||
attrs[4] = f'style="background: {color_good}"'
|
||||
else:
|
||||
attrs[4] = f'style="background: {color_bad}"'
|
||||
else:
|
||||
attrs[4] = ''
|
||||
all_rows.append([row, attrs]);
|
||||
|
||||
if max(errors):
|
||||
all_rows[0][1][0] = "rowspan=3"
|
||||
row = [''] * (len(header))
|
||||
attrs = ['' for x in header]
|
||||
|
||||
attrs[0] = None
|
||||
row[1] = 'errors'
|
||||
row[2] = f'{errors[0]:d}'
|
||||
row[3] = f'{errors[1]:d}'
|
||||
row[4] = '--'
|
||||
row[5] = '--'
|
||||
if errors[0]:
|
||||
attrs[2] += f' style="background: {color_bad}" '
|
||||
if errors[1]:
|
||||
attrs[3] += f' style="background: {color_bad}" '
|
||||
|
||||
all_rows.append([row, attrs])
|
||||
|
||||
text = tableStart('Concurrent benchmarks')
|
||||
text += tableHeader(header)
|
||||
for row, attrs in all_rows:
|
||||
text += tableRow(row, attrs)
|
||||
text += tableEnd()
|
||||
|
||||
global tables
|
||||
tables.append(text)
|
||||
|
||||
try:
|
||||
add_benchmark_results()
|
||||
except:
|
||||
report_errors.append(
|
||||
traceback.format_exception_only(
|
||||
*sys.exc_info()[:2])[-1])
|
||||
pass
|
||||
|
||||
addSimpleTable('Metric changes',
|
||||
['Metric', 'Old median value', 'New median value',
|
||||
'Relative difference', 'Times difference'],
|
||||
|
Loading…
Reference in New Issue
Block a user