2020-01-23 17:48:26 +00:00
#!/usr/bin/python3
2020-04-02 18:44:58 +00:00
import argparse
2020-03-24 17:33:18 +00:00
import ast
2020-01-23 17:48:26 +00:00
import collections
import csv
2020-02-27 17:57:08 +00:00
import itertools
2020-06-11 21:24:56 +00:00
import json
2020-01-23 17:48:26 +00:00
import os
2020-06-29 14:09:51 +00:00
import os . path
2020-06-22 12:22:09 +00:00
import pprint
2020-01-23 17:48:26 +00:00
import sys
2020-02-26 16:18:58 +00:00
import traceback
2020-01-23 17:48:26 +00:00
2020-04-02 18:44:58 +00:00
parser = argparse . ArgumentParser ( description = ' Create performance test report ' )
parser . add_argument ( ' --report ' , default = ' main ' , choices = [ ' main ' , ' all-queries ' ] ,
help = ' Which report to build ' )
args = parser . parse_args ( )
2020-08-05 15:46:05 +00:00
tables = [ ]
2020-08-05 16:07:37 +00:00
errors_explained = [ ]
2020-02-26 16:18:58 +00:00
report_errors = [ ]
2020-02-27 19:43:43 +00:00
error_tests = 0
slow_average_tests = 0
faster_queries = 0
slower_queries = 0
unstable_queries = 0
2020-03-24 17:33:18 +00:00
very_unstable_queries = 0
2020-06-27 00:45:00 +00:00
unstable_partial_queries = 0
2020-02-26 16:18:58 +00:00
2020-04-29 17:26:28 +00:00
# max seconds to run one query by itself, not counting preparation
allowed_single_run_time = 2
2020-05-20 02:19:19 +00:00
color_bad = ' #ffb0c0 '
color_good = ' #b0d050 '
2020-04-02 18:44:58 +00:00
header_template = """
2020-01-23 17:48:26 +00:00
< ! DOCTYPE html >
2020-08-28 22:13:46 +00:00
< html lang = " en " >
2020-08-28 22:46:04 +00:00
< link rel = " preload " as = " font " href = " https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2 " type = " font/woff2 " crossorigin = " anonymous " / >
< style >
2020-01-23 17:48:26 +00:00
@font - face { {
font - family : ' Yandex Sans Display Web ' ;
src : url ( https : / / yastatic . net / adv - www / _ / H63jN0veW07XQUIA2317lr9UIm8 . eot ) ;
src : url ( https : / / yastatic . net / adv - www / _ / H63jN0veW07XQUIA2317lr9UIm8 . eot ? #iefix) format('embedded-opentype'),
url ( https : / / yastatic . net / adv - www / _ / sUYVCPUAQE7ExrvMS7FoISoO83s . woff2 ) format ( ' woff2 ' ) ,
url ( https : / / yastatic . net / adv - www / _ / v2Sve_obH3rKm6rKrtSQpf - eB7U . woff ) format ( ' woff ' ) ,
url ( https : / / yastatic . net / adv - www / _ / PzD8hWLMunow5i3RfJ6WQJAL7aI . ttf ) format ( ' truetype ' ) ,
url ( https : / / yastatic . net / adv - www / _ / lF_KG5g4tpQNlYIgA0e77fBSZ5s . svg #YandexSansDisplayWeb-Regular) format('svg');
font - weight : 400 ;
font - style : normal ;
2020-08-28 22:13:46 +00:00
font - stretch : normal ;
font - display : swap ;
2020-01-23 17:48:26 +00:00
} }
2020-07-31 19:58:18 +00:00
body { {
font - family : " Yandex Sans Display Web " , Arial , sans - serif ;
background : #EEE;
} }
2020-01-23 17:48:26 +00:00
a { { color : #06F; text-decoration: none; }}
2020-07-31 19:58:18 +00:00
2020-01-23 17:48:26 +00:00
a : hover , a : active { { color : #F40; text-decoration: underline; }}
2020-07-31 19:58:18 +00:00
2020-05-09 15:50:28 +00:00
. main { { margin : auto ; max - width : 95 % ; } }
2020-02-27 17:57:08 +00:00
2020-07-31 19:58:18 +00:00
p . links a { {
padding : 5 px ; margin : 3 px ; background : #FFF; line-height: 2;
white - space : nowrap ;
box - shadow : 0 0 0 1 px rgba ( 0 , 0 , 0 , 0.05 ) , 0 8 px 25 px - 5 px rgba ( 0 , 0 , 0 , 0.1 ) ;
} }
. cancela , . cancela : link , . cancela : visited , . cancela : hover ,
. cancela : focus , . cancela : active { {
2020-02-10 16:34:07 +00:00
color : inherit ;
text - decoration : none ;
} }
2020-07-31 10:59:15 +00:00
2020-07-31 19:58:18 +00:00
table { {
border : none ;
border - spacing : 0 px ;
line - height : 1.5 ;
box - shadow : 0 0 0 1 px rgba ( 0 , 0 , 0 , 0.05 ) , 0 8 px 25 px - 5 px rgba ( 0 , 0 , 0 , 0.1 ) ;
text - align : left ;
} }
th , td { {
border : none ;
padding : 5 px ;
vertical - align : top ;
background - color : #FFF;
font - family : sans - serif ;
} }
th { {
border - bottom : 2 px solid black ;
} }
tr : nth - child ( odd ) td { { filter : brightness ( 90 % ) ; } }
. all - query - times tr : nth - child ( 1 ) ,
. all - query - times tr : nth - child ( 2 ) ,
. all - query - times tr : nth - child ( 3 ) ,
. all - query - times tr : nth - child ( 4 ) ,
. all - query - times tr : nth - child ( 5 ) ,
. all - query - times tr : nth - child ( 7 ) ,
. changes - in - performance tr : nth - child ( 1 ) ,
. changes - in - performance tr : nth - child ( 2 ) ,
. changes - in - performance tr : nth - child ( 3 ) ,
. changes - in - performance tr : nth - child ( 4 ) ,
. changes - in - performance tr : nth - child ( 5 ) ,
. changes - in - performance tr : nth - child ( 7 ) ,
. unstable - queries tr : nth - child ( 1 ) ,
. unstable - queries tr : nth - child ( 2 ) ,
. unstable - queries tr : nth - child ( 3 ) ,
. unstable - queries tr : nth - child ( 4 ) ,
. unstable - queries tr : nth - child ( 6 ) ,
. test - performance - changes tr : nth - child ( 2 ) ,
. test - performance - changes tr : nth - child ( 3 ) ,
. test - performance - changes tr : nth - child ( 4 ) ,
. test - performance - changes tr : nth - child ( 5 ) ,
. test - performance - changes tr : nth - child ( 6 ) ,
. test - times tr : nth - child ( 2 ) ,
. test - times tr : nth - child ( 3 ) ,
. test - times tr : nth - child ( 4 ) ,
. test - times tr : nth - child ( 5 ) ,
. test - times tr : nth - child ( 6 ) ,
. test - times tr : nth - child ( 7 ) ,
. test - times tr : nth - child ( 8 ) ,
. concurrent - benchmarks tr : nth - child ( 2 ) ,
. concurrent - benchmarks tr : nth - child ( 3 ) ,
. concurrent - benchmarks tr : nth - child ( 4 ) ,
. concurrent - benchmarks tr : nth - child ( 5 ) ,
. metric - changes tr : nth - child ( 2 ) ,
. metric - changes tr : nth - child ( 3 ) ,
. metric - changes tr : nth - child ( 4 ) ,
. metric - changes tr : nth - child ( 5 )
2020-07-31 10:59:15 +00:00
{ { text - align : right ; } }
2020-01-23 17:48:26 +00:00
< / style >
2020-02-26 16:18:58 +00:00
< title > Clickhouse performance comparison < / title >
2020-01-23 17:48:26 +00:00
< / head >
< body >
< div class = " main " >
2020-02-26 16:18:58 +00:00
< h1 > ClickHouse performance comparison < / h1 >
2020-04-02 18:44:58 +00:00
"""
2020-02-26 16:18:58 +00:00
2020-02-10 16:34:07 +00:00
table_anchor = 0
row_anchor = 0
2020-08-05 15:46:05 +00:00
def currentTableAnchor ( ) :
global table_anchor
return f ' { table_anchor } '
def newTableAnchor ( ) :
2020-02-10 16:34:07 +00:00
global table_anchor
table_anchor + = 1
2020-08-05 15:46:05 +00:00
return currentTableAnchor ( )
def currentRowAnchor ( ) :
global row_anchor
global table_anchor
return f ' { table_anchor } . { row_anchor } '
2020-02-10 16:34:07 +00:00
def nextRowAnchor ( ) :
2020-08-05 15:46:05 +00:00
global row_anchor
global table_anchor
return f ' { table_anchor } . { row_anchor + 1 } '
def advanceRowAnchor ( ) :
2020-02-10 16:34:07 +00:00
global row_anchor
global table_anchor
row_anchor + = 1
2020-08-05 15:46:05 +00:00
return currentRowAnchor ( )
2020-02-10 16:34:07 +00:00
2020-08-07 01:25:45 +00:00
def tr ( x , anchor = None ) :
2020-02-11 20:00:53 +00:00
#return '<tr onclick="location.href=\'#{a}\'" id={a}>{x}</tr>'.format(a=a, x=str(x))
2020-08-07 01:25:45 +00:00
anchor = anchor if anchor else advanceRowAnchor ( )
return f ' <tr id= { anchor } > { x } </tr> '
2020-01-23 17:48:26 +00:00
2020-02-27 17:57:08 +00:00
def td ( value , cell_attributes = ' ' ) :
return ' <td {cell_attributes} > {value} </td> ' . format (
cell_attributes = cell_attributes ,
2020-02-26 16:18:58 +00:00
value = value )
2020-01-23 17:48:26 +00:00
def th ( x ) :
return ' <th> ' + str ( x ) + ' </th> '
2020-08-07 01:25:45 +00:00
def tableRow ( cell_values , cell_attributes = [ ] , anchor = None ) :
return tr (
' ' . join ( [ td ( v , a )
for v , a in itertools . zip_longest (
cell_values , cell_attributes ,
fillvalue = ' ' )
if a is not None and v is not None ] ) ,
anchor )
2020-01-23 17:48:26 +00:00
2020-02-26 16:18:58 +00:00
def tableHeader ( r ) :
2020-01-23 17:48:26 +00:00
return tr ( ' ' . join ( [ th ( f ) for f in r ] ) )
2020-02-26 16:18:58 +00:00
def tableStart ( title ) :
2020-07-31 10:59:15 +00:00
cls = ' - ' . join ( title . lower ( ) . split ( ' ' ) [ : 3 ] ) ;
2020-08-05 15:46:05 +00:00
global table_anchor
table_anchor = cls
anchor = currentTableAnchor ( )
2020-07-31 10:59:15 +00:00
return f """
< h2 id = " {anchor} " >
< a class = " cancela " href = " # {anchor} " > { title } < / a >
< / h2 >
< table class = " {cls} " >
"""
2020-02-26 16:18:58 +00:00
def tableEnd ( ) :
return ' </table> '
2020-02-27 17:57:08 +00:00
def tsvRows ( n ) :
2020-02-26 16:18:58 +00:00
result = [ ]
try :
with open ( n , encoding = ' utf-8 ' ) as fd :
return [ row for row in csv . reader ( fd , delimiter = " \t " , quotechar = ' " ' ) ]
except :
report_errors . append (
traceback . format_exception_only (
* sys . exc_info ( ) [ : 2 ] ) [ - 1 ] )
pass
return [ ]
2020-02-27 17:57:08 +00:00
def htmlRows ( n ) :
rawRows = tsvRows ( n )
2020-01-23 17:48:26 +00:00
result = ' '
2020-02-26 16:18:58 +00:00
for row in rawRows :
result + = tableRow ( row )
2020-01-23 17:48:26 +00:00
return result
2020-08-05 15:46:05 +00:00
def addSimpleTable ( caption , columns , rows , pos = None ) :
global tables
text = ' '
2020-02-27 17:57:08 +00:00
if not rows :
return
2020-08-05 15:46:05 +00:00
text + = tableStart ( caption )
text + = tableHeader ( columns )
2020-02-27 17:57:08 +00:00
for row in rows :
2020-08-05 15:46:05 +00:00
text + = tableRow ( row )
text + = tableEnd ( )
tables . insert ( pos if pos else len ( tables ) , text )
2020-02-27 17:57:08 +00:00
2020-08-05 15:46:05 +00:00
def add_tested_commits ( ) :
2020-04-30 08:36:33 +00:00
global report_errors
try :
2020-08-05 15:46:05 +00:00
addSimpleTable ( ' Tested commits ' , [ ' Old ' , ' New ' ] ,
2020-04-30 08:36:33 +00:00
[ [ ' <pre> {} </pre> ' . format ( x ) for x in
[ open ( ' left-commit.txt ' ) . read ( ) ,
open ( ' right-commit.txt ' ) . read ( ) ] ] ] )
except :
# Don't fail if no commit info -- maybe it's a manual run.
report_errors . append (
traceback . format_exception_only (
* sys . exc_info ( ) [ : 2 ] ) [ - 1 ] )
pass
2020-08-05 15:46:05 +00:00
def add_report_errors ( ) :
global tables
2020-04-30 08:36:33 +00:00
global report_errors
# Add the errors reported by various steps of comparison script
try :
report_errors + = [ l . strip ( ) for l in open ( ' report/errors.log ' ) ]
except :
report_errors . append (
traceback . format_exception_only (
* sys . exc_info ( ) [ : 2 ] ) [ - 1 ] )
pass
2020-08-05 15:46:05 +00:00
if not report_errors :
return
text = tableStart ( ' Errors while building the report ' )
text + = tableHeader ( [ ' Error ' ] )
for x in report_errors :
text + = tableRow ( [ x ] )
text + = tableEnd ( )
# Insert after Tested Commits
tables . insert ( 1 , text )
2020-08-05 16:07:37 +00:00
errors_explained . append ( [ f ' <a href= " # { currentTableAnchor ( ) } " >There were some errors while building the report</a> ' ] ) ;
2020-08-05 15:46:05 +00:00
def add_errors_explained ( ) :
2020-08-06 13:28:51 +00:00
if not errors_explained :
return
2020-08-07 01:25:45 +00:00
text = ' <a name= " fail1 " /> '
text + = tableStart ( ' Error summary ' )
2020-08-06 13:28:51 +00:00
text + = tableHeader ( [ ' Description ' ] )
for row in errors_explained :
text + = tableRow ( row )
text + = tableEnd ( )
2020-08-05 15:46:05 +00:00
global tables
2020-08-06 13:28:51 +00:00
tables . insert ( 1 , text )
2020-08-05 15:46:05 +00:00
2020-04-30 08:36:33 +00:00
2020-04-02 18:44:58 +00:00
if args . report == ' main ' :
print ( header_template . format ( ) )
2020-08-05 15:46:05 +00:00
add_tested_commits ( )
2020-04-02 18:44:58 +00:00
2020-05-22 02:51:26 +00:00
run_error_rows = tsvRows ( ' run-errors.tsv ' )
error_tests + = len ( run_error_rows )
2020-08-05 15:46:05 +00:00
addSimpleTable ( ' Run errors ' , [ ' Test ' , ' Error ' ] , run_error_rows )
if run_error_rows :
2020-08-05 16:07:37 +00:00
errors_explained . append ( [ f ' <a href= " # { currentTableAnchor ( ) } " >There were some errors while running the tests</a> ' ] ) ;
2020-08-05 15:46:05 +00:00
2020-05-22 02:51:26 +00:00
slow_on_client_rows = tsvRows ( ' report/slow-on-client.tsv ' )
error_tests + = len ( slow_on_client_rows )
2020-08-05 15:46:05 +00:00
addSimpleTable ( ' Slow on client ' ,
2020-06-25 20:19:27 +00:00
[ ' Client time, s ' , ' Server time, s ' , ' Ratio ' , ' Test ' , ' Query ' ] ,
2020-05-22 02:51:26 +00:00
slow_on_client_rows )
2020-08-05 15:46:05 +00:00
if slow_on_client_rows :
2020-08-05 16:07:37 +00:00
errors_explained . append ( [ f ' <a href= " # { currentTableAnchor ( ) } " >Some queries are taking noticeable time client-side (missing `FORMAT Null`?)</a> ' ] ) ;
2020-06-23 12:09:54 +00:00
unmarked_short_rows = tsvRows ( ' report/unmarked-short-queries.tsv ' )
error_tests + = len ( unmarked_short_rows )
2020-08-05 15:46:05 +00:00
addSimpleTable ( ' Short queries not marked as short ' ,
2020-06-23 12:09:54 +00:00
[ ' New client time, s ' , ' Test ' , ' # ' , ' Query ' ] ,
unmarked_short_rows )
2020-08-05 15:46:05 +00:00
if unmarked_short_rows :
2020-08-05 16:07:37 +00:00
errors_explained . append ( [ f ' <a href= " # { currentTableAnchor ( ) } " >Some queries have short duration but are not explicitly marked as " short " </a> ' ] ) ;
2020-05-22 02:51:26 +00:00
2020-08-05 15:46:05 +00:00
def add_partial ( ) :
2020-06-27 00:45:00 +00:00
rows = tsvRows ( ' report/partial-queries-report.tsv ' )
if not rows :
return
2020-08-05 15:46:05 +00:00
global unstable_partial_queries , slow_average_tests , tables
text = tableStart ( ' Partial queries ' )
2020-06-27 00:45:00 +00:00
columns = [ ' Median time, s ' , ' Relative time variance ' , ' Test ' , ' # ' , ' Query ' ]
2020-08-05 15:46:05 +00:00
text + = tableHeader ( columns )
2020-06-27 00:45:00 +00:00
attrs = [ ' ' for c in columns ]
for row in rows :
2020-08-07 01:25:45 +00:00
anchor = f ' { currentTableAnchor ( ) } . { row [ 2 ] } . { row [ 3 ] } '
2020-06-27 00:45:00 +00:00
if float ( row [ 1 ] ) > 0.10 :
attrs [ 1 ] = f ' style= " background: { color_bad } " '
unstable_partial_queries + = 1
2020-08-07 01:25:45 +00:00
errors_explained . append ( [ f ' <a href= " # { anchor } " >The query no. { row [ 3 ] } of test \' { row [ 2 ] } \' has excessive variance of run time. Keep it below 10%</a> ' ] )
2020-06-27 00:45:00 +00:00
else :
attrs [ 1 ] = ' '
if float ( row [ 0 ] ) > allowed_single_run_time :
attrs [ 0 ] = f ' style= " background: { color_bad } " '
2020-08-12 08:54:46 +00:00
errors_explained . append ( [ f ' <a href= " # { anchor } " >The query no. { row [ 3 ] } of test \' { row [ 2 ] } \' is taking too long to run. Keep the run time below { allowed_single_run_time } seconds " </a> ' ] )
2020-06-27 00:45:00 +00:00
slow_average_tests + = 1
else :
attrs [ 0 ] = ' '
2020-08-07 01:25:45 +00:00
text + = tableRow ( row , attrs , anchor )
2020-08-05 15:46:05 +00:00
text + = tableEnd ( )
tables . append ( text )
2020-06-27 00:45:00 +00:00
2020-08-05 15:46:05 +00:00
add_partial ( )
2020-06-27 00:45:00 +00:00
2020-08-05 15:46:05 +00:00
def add_changes ( ) :
2020-04-28 07:45:35 +00:00
rows = tsvRows ( ' report/changed-perf.tsv ' )
2020-04-02 18:44:58 +00:00
if not rows :
return
2020-08-05 15:46:05 +00:00
global faster_queries , slower_queries , tables
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text = tableStart ( ' Changes in performance ' )
2020-04-02 18:44:58 +00:00
columns = [
2020-06-25 20:19:27 +00:00
' Old, s ' , # 0
' New, s ' , # 1
2020-07-30 22:13:50 +00:00
' Times speedup / slowdown ' , # 2
' Relative difference (new − old) / old ' , # 3
' p < 0.001 threshold ' , # 4
# Failed # 5
' Test ' , # 6
' # ' , # 7
' Query ' , # 8
2020-04-02 18:44:58 +00:00
]
2020-08-05 15:46:05 +00:00
text + = tableHeader ( columns )
2020-04-02 18:44:58 +00:00
attrs = [ ' ' for c in columns ]
2020-08-01 10:10:36 +00:00
attrs [ 5 ] = None
2020-04-02 18:44:58 +00:00
for row in rows :
2020-08-07 01:25:45 +00:00
anchor = f ' { currentTableAnchor ( ) } . { row [ 6 ] } . { row [ 7 ] } '
2020-07-30 22:13:50 +00:00
if int ( row [ 5 ] ) :
if float ( row [ 3 ] ) < 0. :
2020-04-21 18:46:45 +00:00
faster_queries + = 1
2020-07-30 22:13:50 +00:00
attrs [ 2 ] = attrs [ 3 ] = f ' style= " background: { color_good } " '
2020-04-21 18:46:45 +00:00
else :
slower_queries + = 1
2020-07-30 22:13:50 +00:00
attrs [ 2 ] = attrs [ 3 ] = f ' style= " background: { color_bad } " '
2020-08-07 01:25:45 +00:00
errors_explained . append ( [ f ' <a href= " # { anchor } " >The query no. { row [ 7 ] } of test \' { row [ 6 ] } \' has slowed down</a> ' ] )
2020-04-28 07:45:35 +00:00
else :
2020-07-30 22:13:50 +00:00
attrs [ 2 ] = attrs [ 3 ] = ' '
2020-04-02 18:44:58 +00:00
2020-08-07 01:25:45 +00:00
text + = tableRow ( row , attrs , anchor )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text + = tableEnd ( )
tables . append ( text )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_changes ( )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
def add_unstable_queries ( ) :
global unstable_queries , very_unstable_queries , tables
2020-04-02 18:44:58 +00:00
2020-04-28 07:45:35 +00:00
unstable_rows = tsvRows ( ' report/unstable-queries.tsv ' )
2020-04-02 18:44:58 +00:00
if not unstable_rows :
return
unstable_queries + = len ( unstable_rows )
columns = [
2020-06-25 20:19:27 +00:00
' Old, s ' , #0
' New, s ' , #1
2020-04-02 18:44:58 +00:00
' Relative difference (new - old)/old ' , #2
2020-07-31 10:59:15 +00:00
' p < 0.001 threshold ' , #3
2020-04-28 07:45:35 +00:00
# Failed #4
' Test ' , #5
2020-06-08 13:57:33 +00:00
' # ' , #6
' Query ' #7
2020-02-27 17:57:08 +00:00
]
2020-08-05 15:46:05 +00:00
text = tableStart ( ' Unstable queries ' )
text + = tableHeader ( columns )
2020-04-02 18:44:58 +00:00
attrs = [ ' ' for c in columns ]
2020-04-28 07:45:35 +00:00
attrs [ 4 ] = None
2020-04-02 18:44:58 +00:00
for r in unstable_rows :
2020-08-07 01:25:45 +00:00
anchor = f ' { currentTableAnchor ( ) } . { r [ 5 ] } . { r [ 6 ] } '
2020-04-28 07:45:35 +00:00
if int ( r [ 4 ] ) :
2020-04-02 18:44:58 +00:00
very_unstable_queries + = 1
2020-05-20 02:19:19 +00:00
attrs [ 3 ] = f ' style= " background: { color_bad } " '
2020-04-02 18:44:58 +00:00
else :
attrs [ 3 ] = ' '
2020-08-07 01:25:45 +00:00
text + = tableRow ( r , attrs , anchor )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text + = tableEnd ( )
tables . append ( text )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_unstable_queries ( )
2020-04-02 18:44:58 +00:00
2020-05-20 02:19:19 +00:00
skipped_tests_rows = tsvRows ( ' analyze/skipped-tests.tsv ' )
2020-08-05 15:46:05 +00:00
addSimpleTable ( ' Skipped tests ' , [ ' Test ' , ' Reason ' ] , skipped_tests_rows )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
addSimpleTable ( ' Test performance changes ' ,
2020-06-09 13:29:07 +00:00
[ ' Test ' , ' Queries ' , ' Unstable ' , ' Changed perf ' , ' Total not OK ' , ' Avg relative time diff ' ] ,
tsvRows ( ' report/test-perf-changes.tsv ' ) )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
def add_test_times ( ) :
global slow_average_tests , tables
2020-04-28 07:45:35 +00:00
rows = tsvRows ( ' report/test-times.tsv ' )
2020-04-02 18:44:58 +00:00
if not rows :
return
columns = [
' Test ' , #0
2020-06-25 20:19:27 +00:00
' Wall clock time, s ' , #1
' Total client time, s ' , #2
2020-04-02 18:44:58 +00:00
' Total queries ' , #3
' Ignored short queries ' , #4
2020-06-25 20:19:27 +00:00
' Longest query<br>(sum for all runs), s ' , #5
' Avg wall clock time<br>(sum for all runs), s ' , #6
' Shortest query<br>(sum for all runs), s ' , #7
2020-04-02 18:44:58 +00:00
]
2020-08-05 15:46:05 +00:00
text = tableStart ( ' Test times ' )
text + = tableHeader ( columns )
2020-05-09 15:58:47 +00:00
2020-04-29 17:26:28 +00:00
nominal_runs = 13 # FIXME pass this as an argument
total_runs = ( nominal_runs + 1 ) * 2 # one prewarm run, two servers
2020-04-02 18:44:58 +00:00
attrs = [ ' ' for c in columns ]
for r in rows :
2020-09-01 19:05:57 +00:00
anchor = f ' { currentTableAnchor ( ) } . { r [ 0 ] } '
2020-04-29 17:26:28 +00:00
if float ( r [ 6 ] ) > 1.5 * total_runs :
2020-04-02 18:44:58 +00:00
# FIXME should be 15s max -- investigate parallel_insert
slow_average_tests + = 1
2020-05-20 02:19:19 +00:00
attrs [ 6 ] = f ' style= " background: { color_bad } " '
2020-09-01 19:05:57 +00:00
errors_explained . append ( [ f ' <a href= " # { anchor } " >The test \' { r [ 0 ] } \' is too slow to run as a whole. Investigate whether the create and fill queries can be sped up ' ] )
2020-04-02 18:44:58 +00:00
else :
attrs [ 6 ] = ' '
2020-04-29 17:26:28 +00:00
if float ( r [ 5 ] ) > allowed_single_run_time * total_runs :
2020-04-02 18:44:58 +00:00
slow_average_tests + = 1
2020-05-20 02:19:19 +00:00
attrs [ 5 ] = f ' style= " background: { color_bad } " '
2020-08-07 01:25:45 +00:00
errors_explained . append ( [ f ' <a href= " ./all-queries.html#all-query-times. { r [ 0 ] } .0 " >Some query of the test \' { r [ 0 ] } \' is too slow to run. See the all queries report ' ] )
2020-04-02 18:44:58 +00:00
else :
attrs [ 5 ] = ' '
2020-09-01 19:05:57 +00:00
text + = tableRow ( r , attrs , anchor )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text + = tableEnd ( )
tables . append ( text )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_test_times ( )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
def add_benchmark_results ( ) :
2020-06-29 14:09:51 +00:00
if not os . path . isfile ( ' benchmark/website-left.json ' ) :
return
2020-06-19 22:41:15 +00:00
json_reports = [ json . load ( open ( f ' benchmark/website- { x } .json ' ) ) for x in [ ' left ' , ' right ' ] ]
stats = [ next ( iter ( x . values ( ) ) ) [ " statistics " ] for x in json_reports ]
qps = [ x [ " QPS " ] for x in stats ]
2020-06-22 12:22:09 +00:00
queries = [ x [ " num_queries " ] for x in stats ]
2020-06-19 22:41:15 +00:00
errors = [ x [ " num_errors " ] for x in stats ]
relative_diff = ( qps [ 1 ] - qps [ 0 ] ) / max ( 0.01 , qps [ 0 ] ) ;
times_diff = max ( qps ) / max ( 0.01 , min ( qps ) )
2020-06-22 12:22:09 +00:00
2020-06-19 22:41:15 +00:00
all_rows = [ ]
2020-06-22 12:22:09 +00:00
header = [ ' Benchmark ' , ' Metric ' , ' Old ' , ' New ' , ' Relative difference ' , ' Times difference ' ] ;
attrs = [ ' ' for x in header ]
row = [ ' website ' , ' queries ' , f ' { queries [ 0 ] : d } ' , f ' { queries [ 1 ] : d } ' , ' -- ' , ' -- ' ]
attrs [ 0 ] = ' rowspan=2 '
all_rows . append ( [ row , attrs ] )
attrs = [ ' ' for x in header ]
row = [ None , ' queries/s ' , f ' { qps [ 0 ] : .3f } ' , f ' { qps [ 1 ] : .3f } ' , f ' { relative_diff : .3f } ' , f ' x { times_diff : .3f } ' ]
2020-06-11 21:24:56 +00:00
if abs ( relative_diff ) > 0.1 :
# More queries per second is better.
if relative_diff > 0. :
2020-06-22 12:22:09 +00:00
attrs [ 4 ] = f ' style= " background: { color_good } " '
2020-06-11 21:24:56 +00:00
else :
2020-06-22 12:22:09 +00:00
attrs [ 4 ] = f ' style= " background: { color_bad } " '
2020-06-11 21:24:56 +00:00
else :
2020-06-22 12:22:09 +00:00
attrs [ 4 ] = ' '
all_rows . append ( [ row , attrs ] ) ;
2020-06-19 22:41:15 +00:00
if max ( errors ) :
2020-06-22 12:22:09 +00:00
all_rows [ 0 ] [ 1 ] [ 0 ] = " rowspan=3 "
row = [ ' ' ] * ( len ( header ) )
attrs = [ ' ' for x in header ]
attrs [ 0 ] = None
row [ 1 ] = ' errors '
row [ 2 ] = f ' { errors [ 0 ] : d } '
row [ 3 ] = f ' { errors [ 1 ] : d } '
row [ 4 ] = ' -- '
row [ 5 ] = ' -- '
2020-06-19 22:41:15 +00:00
if errors [ 0 ] :
attrs [ 2 ] + = f ' style= " background: { color_bad } " '
2020-06-22 12:22:09 +00:00
if errors [ 1 ] :
attrs [ 3 ] + = f ' style= " background: { color_bad } " '
2020-06-19 22:41:15 +00:00
2020-06-22 12:22:09 +00:00
all_rows . append ( [ row , attrs ] )
2020-06-19 22:41:15 +00:00
2020-08-05 15:46:05 +00:00
text = tableStart ( ' Concurrent benchmarks ' )
text + = tableHeader ( header )
2020-06-19 22:41:15 +00:00
for row , attrs in all_rows :
2020-08-05 15:46:05 +00:00
text + = tableRow ( row , attrs )
text + = tableEnd ( )
global tables
tables . append ( text )
2020-06-11 21:24:56 +00:00
try :
2020-08-05 15:46:05 +00:00
add_benchmark_results ( )
2020-06-11 21:24:56 +00:00
except :
report_errors . append (
traceback . format_exception_only (
* sys . exc_info ( ) [ : 2 ] ) [ - 1 ] )
pass
2020-08-05 15:46:05 +00:00
addSimpleTable ( ' Metric changes ' ,
2020-06-25 20:19:27 +00:00
[ ' Metric ' , ' Old median value ' , ' New median value ' ,
' Relative difference ' , ' Times difference ' ] ,
tsvRows ( ' metrics/changes.tsv ' ) )
2020-08-05 15:46:05 +00:00
add_report_errors ( )
add_errors_explained ( )
for t in tables :
print ( t )
2020-04-02 18:44:58 +00:00
print ( """
2020-08-28 22:46:04 +00:00
< / div >
2020-04-02 18:44:58 +00:00
< p class = " links " >
< a href = " all-queries.html " > All queries < / a >
< a href = " compare.log " > Log < / a >
2020-05-20 02:19:19 +00:00
< a href = " output.7z " > Test output < / a >
2020-04-02 18:44:58 +00:00
< / p >
< / body >
< / html >
""" )
status = ' success '
message = ' See the report '
message_array = [ ]
if slow_average_tests :
status = ' failure '
message_array . append ( str ( slow_average_tests ) + ' too long ' )
if faster_queries :
message_array . append ( str ( faster_queries ) + ' faster ' )
if slower_queries :
2020-05-13 03:55:19 +00:00
if slower_queries > 3 :
status = ' failure '
2020-04-02 18:44:58 +00:00
message_array . append ( str ( slower_queries ) + ' slower ' )
2020-06-27 00:45:00 +00:00
if unstable_partial_queries :
unstable_queries + = unstable_partial_queries
error_tests + = unstable_partial_queries
status = ' failure '
2020-04-02 18:44:58 +00:00
if unstable_queries :
message_array . append ( str ( unstable_queries ) + ' unstable ' )
2020-05-09 15:58:47 +00:00
# Disabled before fix.
# if very_unstable_queries:
# status = 'failure'
2020-04-02 18:44:58 +00:00
error_tests + = slow_average_tests
if error_tests :
status = ' failure '
2020-06-10 21:38:26 +00:00
message_array . insert ( 0 , str ( error_tests ) + ' errors ' )
2020-04-02 18:44:58 +00:00
if message_array :
message = ' , ' . join ( message_array )
if report_errors :
status = ' failure '
message = ' Errors while building the report. '
print ( """
< ! - - status : { status } - - >
< ! - - message : { message } - - >
""" .format(status=status, message=message))
elif args . report == ' all-queries ' :
print ( header_template . format ( ) )
2020-08-05 15:46:05 +00:00
add_tested_commits ( )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
def add_all_queries ( ) :
2020-04-28 07:45:35 +00:00
rows = tsvRows ( ' report/all-queries.tsv ' )
2020-04-02 18:44:58 +00:00
if not rows :
return
columns = [
2020-04-28 07:45:35 +00:00
# Changed #0
# Unstable #1
2020-06-25 20:19:27 +00:00
' Old, s ' , #2
' New, s ' , #3
2020-07-30 22:13:50 +00:00
' Times speedup / slowdown ' , #4
' Relative difference (new − old) / old ' , #5
2020-07-31 10:59:15 +00:00
' p < 0.001 threshold ' , #6
2020-04-28 07:45:35 +00:00
' Test ' , #7
2020-06-08 13:57:33 +00:00
' # ' , #8
' Query ' , #9
2020-04-02 18:44:58 +00:00
]
2020-08-05 15:46:05 +00:00
text = tableStart ( ' All query times ' )
text + = tableHeader ( columns )
2020-04-02 18:44:58 +00:00
attrs = [ ' ' for c in columns ]
2020-04-28 07:45:35 +00:00
attrs [ 0 ] = None
attrs [ 1 ] = None
2020-04-02 18:44:58 +00:00
for r in rows :
2020-08-07 01:25:45 +00:00
anchor = f ' { currentTableAnchor ( ) } . { r [ 7 ] } . { r [ 8 ] } '
2020-04-28 07:45:35 +00:00
if int ( r [ 1 ] ) :
2020-05-20 02:19:19 +00:00
attrs [ 6 ] = f ' style= " background: { color_bad } " '
2020-04-21 18:46:45 +00:00
else :
2020-04-28 07:45:35 +00:00
attrs [ 6 ] = ' '
2020-04-21 18:46:45 +00:00
2020-04-28 07:45:35 +00:00
if int ( r [ 0 ] ) :
2020-07-30 22:13:50 +00:00
if float ( r [ 5 ] ) > 0. :
attrs [ 4 ] = attrs [ 5 ] = f ' style= " background: { color_bad } " '
2020-04-21 18:46:45 +00:00
else :
2020-07-30 22:13:50 +00:00
attrs [ 4 ] = attrs [ 5 ] = f ' style= " background: { color_good } " '
2020-04-02 18:44:58 +00:00
else :
2020-07-30 22:13:50 +00:00
attrs [ 4 ] = attrs [ 5 ] = ' '
2020-04-02 18:44:58 +00:00
2020-04-29 17:26:28 +00:00
if ( float ( r [ 2 ] ) + float ( r [ 3 ] ) ) / 2 > allowed_single_run_time :
2020-05-20 02:19:19 +00:00
attrs [ 2 ] = f ' style= " background: { color_bad } " '
attrs [ 3 ] = f ' style= " background: { color_bad } " '
2020-04-29 17:26:28 +00:00
else :
attrs [ 2 ] = ' '
attrs [ 3 ] = ' '
2020-08-07 01:25:45 +00:00
text + = tableRow ( r , attrs , anchor )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text + = tableEnd ( )
tables . append ( text )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_all_queries ( )
add_report_errors ( )
for t in tables :
print ( t )
2020-04-30 08:36:33 +00:00
2020-04-02 18:44:58 +00:00
print ( """
2020-08-28 22:46:04 +00:00
< / div >
2020-04-02 18:44:58 +00:00
< p class = " links " >
< a href = " report.html " > Main report < / a >
< a href = " compare.log " > Log < / a >
2020-05-20 02:19:19 +00:00
< a href = " output.7z " > Test output < / a >
2020-04-02 18:44:58 +00:00
< / p >
< / body >
< / html >
""" )
