2020-01-23 17:48:26 +00:00
#!/usr/bin/python3
2020-04-02 18:44:58 +00:00
import argparse
2020-03-24 17:33:18 +00:00
import ast
2020-01-23 17:48:26 +00:00
import collections
import csv
2020-02-27 17:57:08 +00:00
import itertools
2020-06-11 21:24:56 +00:00
import json
2020-01-23 17:48:26 +00:00
import os
2020-06-29 14:09:51 +00:00
import os . path
2020-06-22 12:22:09 +00:00
import pprint
2020-01-23 17:48:26 +00:00
import sys
2020-02-26 16:18:58 +00:00
import traceback
2020-01-23 17:48:26 +00:00
2020-04-02 18:44:58 +00:00
parser = argparse . ArgumentParser ( description = ' Create performance test report ' )
parser . add_argument ( ' --report ' , default = ' main ' , choices = [ ' main ' , ' all-queries ' ] ,
help = ' Which report to build ' )
args = parser . parse_args ( )
2020-08-05 15:46:05 +00:00
tables = [ ]
2020-08-05 16:07:37 +00:00
errors_explained = [ ]
2020-02-26 16:18:58 +00:00
report_errors = [ ]
2020-02-27 19:43:43 +00:00
error_tests = 0
slow_average_tests = 0
faster_queries = 0
slower_queries = 0
unstable_queries = 0
2020-03-24 17:33:18 +00:00
very_unstable_queries = 0
2020-06-27 00:45:00 +00:00
unstable_partial_queries = 0
2020-02-26 16:18:58 +00:00
2020-04-29 17:26:28 +00:00
# max seconds to run one query by itself, not counting preparation
allowed_single_run_time = 2
2020-05-20 02:19:19 +00:00
color_bad = ' #ffb0c0 '
color_good = ' #b0d050 '
2020-04-02 18:44:58 +00:00
header_template = """
2020-01-23 17:48:26 +00:00
< ! DOCTYPE html >
2020-08-28 22:13:46 +00:00
< html lang = " en " >
2020-08-28 22:46:04 +00:00
< link rel = " preload " as = " font " href = " https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2 " type = " font/woff2 " crossorigin = " anonymous " / >
< style >
2020-01-23 17:48:26 +00:00
@font - face { {
font - family : ' Yandex Sans Display Web ' ;
src : url ( https : / / yastatic . net / adv - www / _ / H63jN0veW07XQUIA2317lr9UIm8 . eot ) ;
src : url ( https : / / yastatic . net / adv - www / _ / H63jN0veW07XQUIA2317lr9UIm8 . eot ? #iefix) format('embedded-opentype'),
url ( https : / / yastatic . net / adv - www / _ / sUYVCPUAQE7ExrvMS7FoISoO83s . woff2 ) format ( ' woff2 ' ) ,
url ( https : / / yastatic . net / adv - www / _ / v2Sve_obH3rKm6rKrtSQpf - eB7U . woff ) format ( ' woff ' ) ,
url ( https : / / yastatic . net / adv - www / _ / PzD8hWLMunow5i3RfJ6WQJAL7aI . ttf ) format ( ' truetype ' ) ,
url ( https : / / yastatic . net / adv - www / _ / lF_KG5g4tpQNlYIgA0e77fBSZ5s . svg #YandexSansDisplayWeb-Regular) format('svg');
font - weight : 400 ;
font - style : normal ;
2020-08-28 22:13:46 +00:00
font - stretch : normal ;
font - display : swap ;
2020-01-23 17:48:26 +00:00
} }
2020-07-31 19:58:18 +00:00
body { {
font - family : " Yandex Sans Display Web " , Arial , sans - serif ;
background : #EEE;
} }
2020-01-23 17:48:26 +00:00
a { { color : #06F; text-decoration: none; }}
2020-07-31 19:58:18 +00:00
2020-01-23 17:48:26 +00:00
a : hover , a : active { { color : #F40; text-decoration: underline; }}
2020-07-31 19:58:18 +00:00
2020-05-09 15:50:28 +00:00
. main { { margin : auto ; max - width : 95 % ; } }
2020-02-27 17:57:08 +00:00
2020-07-31 19:58:18 +00:00
p . links a { {
padding : 5 px ; margin : 3 px ; background : #FFF; line-height: 2;
white - space : nowrap ;
box - shadow : 0 0 0 1 px rgba ( 0 , 0 , 0 , 0.05 ) , 0 8 px 25 px - 5 px rgba ( 0 , 0 , 0 , 0.1 ) ;
} }
. cancela , . cancela : link , . cancela : visited , . cancela : hover ,
. cancela : focus , . cancela : active { {
2020-02-10 16:34:07 +00:00
color : inherit ;
text - decoration : none ;
} }
2020-07-31 10:59:15 +00:00
2020-07-31 19:58:18 +00:00
table { {
border : none ;
border - spacing : 0 px ;
line - height : 1.5 ;
box - shadow : 0 0 0 1 px rgba ( 0 , 0 , 0 , 0.05 ) , 0 8 px 25 px - 5 px rgba ( 0 , 0 , 0 , 0.1 ) ;
text - align : left ;
} }
th , td { {
border : none ;
padding : 5 px ;
vertical - align : top ;
background - color : #FFF;
font - family : sans - serif ;
} }
th { {
border - bottom : 2 px solid black ;
} }
tr : nth - child ( odd ) td { { filter : brightness ( 90 % ) ; } }
2020-09-21 12:14:47 +00:00
. unexpected - query - duration tr : nth - child ( 2 ) ,
. unexpected - query - duration tr : nth - child ( 3 ) ,
. unexpected - query - duration tr : nth - child ( 5 ) ,
2020-07-31 19:58:18 +00:00
. all - query - times tr : nth - child ( 1 ) ,
. all - query - times tr : nth - child ( 2 ) ,
. all - query - times tr : nth - child ( 3 ) ,
. all - query - times tr : nth - child ( 4 ) ,
. all - query - times tr : nth - child ( 5 ) ,
. all - query - times tr : nth - child ( 7 ) ,
. changes - in - performance tr : nth - child ( 1 ) ,
. changes - in - performance tr : nth - child ( 2 ) ,
. changes - in - performance tr : nth - child ( 3 ) ,
. changes - in - performance tr : nth - child ( 4 ) ,
. changes - in - performance tr : nth - child ( 5 ) ,
. changes - in - performance tr : nth - child ( 7 ) ,
. unstable - queries tr : nth - child ( 1 ) ,
. unstable - queries tr : nth - child ( 2 ) ,
. unstable - queries tr : nth - child ( 3 ) ,
. unstable - queries tr : nth - child ( 4 ) ,
. unstable - queries tr : nth - child ( 6 ) ,
. test - performance - changes tr : nth - child ( 2 ) ,
. test - performance - changes tr : nth - child ( 3 ) ,
. test - performance - changes tr : nth - child ( 4 ) ,
. test - performance - changes tr : nth - child ( 5 ) ,
. test - performance - changes tr : nth - child ( 6 ) ,
. test - times tr : nth - child ( 2 ) ,
. test - times tr : nth - child ( 3 ) ,
. test - times tr : nth - child ( 4 ) ,
. test - times tr : nth - child ( 5 ) ,
. test - times tr : nth - child ( 6 ) ,
. test - times tr : nth - child ( 7 ) ,
. concurrent - benchmarks tr : nth - child ( 2 ) ,
. concurrent - benchmarks tr : nth - child ( 3 ) ,
. concurrent - benchmarks tr : nth - child ( 4 ) ,
. concurrent - benchmarks tr : nth - child ( 5 ) ,
. metric - changes tr : nth - child ( 2 ) ,
. metric - changes tr : nth - child ( 3 ) ,
. metric - changes tr : nth - child ( 4 ) ,
. metric - changes tr : nth - child ( 5 )
2020-07-31 10:59:15 +00:00
{ { text - align : right ; } }
2020-01-23 17:48:26 +00:00
< / style >
2020-02-26 16:18:58 +00:00
< title > Clickhouse performance comparison < / title >
2020-01-23 17:48:26 +00:00
< / head >
< body >
< div class = " main " >
2020-02-26 16:18:58 +00:00
< h1 > ClickHouse performance comparison < / h1 >
2020-04-02 18:44:58 +00:00
"""
2020-02-26 16:18:58 +00:00
2020-02-10 16:34:07 +00:00
table_anchor = 0
row_anchor = 0
2020-08-05 15:46:05 +00:00
def currentTableAnchor ( ) :
global table_anchor
return f ' { table_anchor } '
def newTableAnchor ( ) :
2020-02-10 16:34:07 +00:00
global table_anchor
table_anchor + = 1
2020-08-05 15:46:05 +00:00
return currentTableAnchor ( )
def currentRowAnchor ( ) :
global row_anchor
global table_anchor
return f ' { table_anchor } . { row_anchor } '
2020-02-10 16:34:07 +00:00
def nextRowAnchor ( ) :
2020-08-05 15:46:05 +00:00
global row_anchor
global table_anchor
return f ' { table_anchor } . { row_anchor + 1 } '
def advanceRowAnchor ( ) :
2020-02-10 16:34:07 +00:00
global row_anchor
global table_anchor
row_anchor + = 1
2020-08-05 15:46:05 +00:00
return currentRowAnchor ( )
2020-02-10 16:34:07 +00:00
2020-08-07 01:25:45 +00:00
def tr ( x , anchor = None ) :
2020-02-11 20:00:53 +00:00
#return '<tr onclick="location.href=\'#{a}\'" id={a}>{x}</tr>'.format(a=a, x=str(x))
2020-08-07 01:25:45 +00:00
anchor = anchor if anchor else advanceRowAnchor ( )
return f ' <tr id= { anchor } > { x } </tr> '
2020-01-23 17:48:26 +00:00
2020-02-27 17:57:08 +00:00
def td ( value , cell_attributes = ' ' ) :
return ' <td {cell_attributes} > {value} </td> ' . format (
cell_attributes = cell_attributes ,
2020-02-26 16:18:58 +00:00
value = value )
2020-01-23 17:48:26 +00:00
2020-09-23 20:45:39 +00:00
def th ( value , cell_attributes = ' ' ) :
return ' <th {cell_attributes} > {value} </th> ' . format (
cell_attributes = cell_attributes ,
value = value )
2020-01-23 17:48:26 +00:00
2020-08-07 01:25:45 +00:00
def tableRow ( cell_values , cell_attributes = [ ] , anchor = None ) :
return tr (
' ' . join ( [ td ( v , a )
for v , a in itertools . zip_longest (
cell_values , cell_attributes ,
fillvalue = ' ' )
if a is not None and v is not None ] ) ,
anchor )
2020-01-23 17:48:26 +00:00
2020-09-23 20:45:39 +00:00
def tableHeader ( cell_values , cell_attributes = [ ] ) :
return tr (
' ' . join ( [ th ( v , a )
for v , a in itertools . zip_longest (
cell_values , cell_attributes ,
fillvalue = ' ' )
if a is not None and v is not None ] ) )
2020-01-23 17:48:26 +00:00
2020-02-26 16:18:58 +00:00
def tableStart ( title ) :
2020-07-31 10:59:15 +00:00
cls = ' - ' . join ( title . lower ( ) . split ( ' ' ) [ : 3 ] ) ;
2020-08-05 15:46:05 +00:00
global table_anchor
table_anchor = cls
anchor = currentTableAnchor ( )
2020-09-17 16:21:59 +00:00
help_anchor = ' - ' . join ( title . lower ( ) . split ( ' ' ) ) ;
2020-07-31 10:59:15 +00:00
return f """
< h2 id = " {anchor} " >
< a class = " cancela " href = " # {anchor} " > { title } < / a >
2020-09-17 16:21:59 +00:00
< a class = " cancela " href = " https://github.com/ClickHouse/ClickHouse/tree/master/docker/test/performance-comparison# {help_anchor} " > < sup style = " color: #888 " > ? < / sup > < / a >
2020-07-31 10:59:15 +00:00
< / h2 >
< table class = " {cls} " >
"""
2020-02-26 16:18:58 +00:00
def tableEnd ( ) :
return ' </table> '
2020-02-27 17:57:08 +00:00
def tsvRows ( n ) :
2020-02-26 16:18:58 +00:00
result = [ ]
try :
with open ( n , encoding = ' utf-8 ' ) as fd :
return [ row for row in csv . reader ( fd , delimiter = " \t " , quotechar = ' " ' ) ]
except :
report_errors . append (
traceback . format_exception_only (
* sys . exc_info ( ) [ : 2 ] ) [ - 1 ] )
pass
return [ ]
2020-02-27 17:57:08 +00:00
def htmlRows ( n ) :
rawRows = tsvRows ( n )
2020-01-23 17:48:26 +00:00
result = ' '
2020-02-26 16:18:58 +00:00
for row in rawRows :
result + = tableRow ( row )
2020-01-23 17:48:26 +00:00
return result
2020-08-05 15:46:05 +00:00
def addSimpleTable ( caption , columns , rows , pos = None ) :
global tables
text = ' '
2020-02-27 17:57:08 +00:00
if not rows :
return
2020-08-05 15:46:05 +00:00
text + = tableStart ( caption )
text + = tableHeader ( columns )
2020-02-27 17:57:08 +00:00
for row in rows :
2020-08-05 15:46:05 +00:00
text + = tableRow ( row )
text + = tableEnd ( )
tables . insert ( pos if pos else len ( tables ) , text )
2020-02-27 17:57:08 +00:00
2020-08-05 15:46:05 +00:00
def add_tested_commits ( ) :
2020-04-30 08:36:33 +00:00
global report_errors
try :
2020-09-17 16:21:59 +00:00
addSimpleTable ( ' Tested Commits ' , [ ' Old ' , ' New ' ] ,
2020-04-30 08:36:33 +00:00
[ [ ' <pre> {} </pre> ' . format ( x ) for x in
[ open ( ' left-commit.txt ' ) . read ( ) ,
open ( ' right-commit.txt ' ) . read ( ) ] ] ] )
except :
# Don't fail if no commit info -- maybe it's a manual run.
report_errors . append (
traceback . format_exception_only (
* sys . exc_info ( ) [ : 2 ] ) [ - 1 ] )
pass
2020-08-05 15:46:05 +00:00
def add_report_errors ( ) :
global tables
2020-04-30 08:36:33 +00:00
global report_errors
# Add the errors reported by various steps of comparison script
try :
report_errors + = [ l . strip ( ) for l in open ( ' report/errors.log ' ) ]
except :
report_errors . append (
traceback . format_exception_only (
* sys . exc_info ( ) [ : 2 ] ) [ - 1 ] )
pass
2020-08-05 15:46:05 +00:00
if not report_errors :
return
2020-09-17 16:21:59 +00:00
text = tableStart ( ' Errors while Building the Report ' )
2020-08-05 15:46:05 +00:00
text + = tableHeader ( [ ' Error ' ] )
for x in report_errors :
text + = tableRow ( [ x ] )
text + = tableEnd ( )
# Insert after Tested Commits
tables . insert ( 1 , text )
2020-08-05 16:07:37 +00:00
errors_explained . append ( [ f ' <a href= " # { currentTableAnchor ( ) } " >There were some errors while building the report</a> ' ] ) ;
2020-08-05 15:46:05 +00:00
def add_errors_explained ( ) :
2020-08-06 13:28:51 +00:00
if not errors_explained :
return
2020-08-07 01:25:45 +00:00
text = ' <a name= " fail1 " /> '
2020-09-17 16:21:59 +00:00
text + = tableStart ( ' Error Summary ' )
2020-08-06 13:28:51 +00:00
text + = tableHeader ( [ ' Description ' ] )
for row in errors_explained :
text + = tableRow ( row )
text + = tableEnd ( )
2020-08-05 15:46:05 +00:00
global tables
2020-08-06 13:28:51 +00:00
tables . insert ( 1 , text )
2020-08-05 15:46:05 +00:00
2020-04-30 08:36:33 +00:00
2020-04-02 18:44:58 +00:00
if args . report == ' main ' :
2020-10-02 16:54:07 +00:00
print ( ( header_template . format ( ) ) )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_tested_commits ( )
2020-04-02 18:44:58 +00:00
2020-05-22 02:51:26 +00:00
run_error_rows = tsvRows ( ' run-errors.tsv ' )
error_tests + = len ( run_error_rows )
2020-09-17 16:21:59 +00:00
addSimpleTable ( ' Run Errors ' , [ ' Test ' , ' Error ' ] , run_error_rows )
2020-08-05 15:46:05 +00:00
if run_error_rows :
2020-08-05 16:07:37 +00:00
errors_explained . append ( [ f ' <a href= " # { currentTableAnchor ( ) } " >There were some errors while running the tests</a> ' ] ) ;
2020-08-05 15:46:05 +00:00
2020-05-22 02:51:26 +00:00
slow_on_client_rows = tsvRows ( ' report/slow-on-client.tsv ' )
error_tests + = len ( slow_on_client_rows )
2020-09-17 16:21:59 +00:00
addSimpleTable ( ' Slow on Client ' ,
2020-06-25 20:19:27 +00:00
[ ' Client time, s ' , ' Server time, s ' , ' Ratio ' , ' Test ' , ' Query ' ] ,
2020-05-22 02:51:26 +00:00
slow_on_client_rows )
2020-08-05 15:46:05 +00:00
if slow_on_client_rows :
2020-08-05 16:07:37 +00:00
errors_explained . append ( [ f ' <a href= " # { currentTableAnchor ( ) } " >Some queries are taking noticeable time client-side (missing `FORMAT Null`?)</a> ' ] ) ;
2020-06-23 12:09:54 +00:00
2020-09-21 12:14:47 +00:00
unmarked_short_rows = tsvRows ( ' report/unexpected-query-duration.tsv ' )
2020-06-23 12:09:54 +00:00
error_tests + = len ( unmarked_short_rows )
2020-09-21 12:14:47 +00:00
addSimpleTable ( ' Unexpected Query Duration ' ,
[ ' Problem ' , ' Marked as " short " ? ' , ' Run time, s ' , ' Test ' , ' # ' , ' Query ' ] ,
2020-06-23 12:09:54 +00:00
unmarked_short_rows )
2020-08-05 15:46:05 +00:00
if unmarked_short_rows :
2020-09-21 12:14:47 +00:00
errors_explained . append ( [ f ' <a href= " # { currentTableAnchor ( ) } " >Some queries have unexpected duration</a> ' ] ) ;
2020-05-22 02:51:26 +00:00
2020-08-05 15:46:05 +00:00
def add_partial ( ) :
2020-06-27 00:45:00 +00:00
rows = tsvRows ( ' report/partial-queries-report.tsv ' )
if not rows :
return
2020-08-05 15:46:05 +00:00
global unstable_partial_queries , slow_average_tests , tables
2020-09-17 16:21:59 +00:00
text = tableStart ( ' Partial Queries ' )
2020-06-27 00:45:00 +00:00
columns = [ ' Median time, s ' , ' Relative time variance ' , ' Test ' , ' # ' , ' Query ' ]
2020-08-05 15:46:05 +00:00
text + = tableHeader ( columns )
2020-06-27 00:45:00 +00:00
attrs = [ ' ' for c in columns ]
for row in rows :
2020-08-07 01:25:45 +00:00
anchor = f ' { currentTableAnchor ( ) } . { row [ 2 ] } . { row [ 3 ] } '
2020-06-27 00:45:00 +00:00
if float ( row [ 1 ] ) > 0.10 :
attrs [ 1 ] = f ' style= " background: { color_bad } " '
unstable_partial_queries + = 1
2020-08-07 01:25:45 +00:00
errors_explained . append ( [ f ' <a href= " # { anchor } " >The query no. { row [ 3 ] } of test \' { row [ 2 ] } \' has excessive variance of run time. Keep it below 10%</a> ' ] )
2020-06-27 00:45:00 +00:00
else :
attrs [ 1 ] = ' '
if float ( row [ 0 ] ) > allowed_single_run_time :
attrs [ 0 ] = f ' style= " background: { color_bad } " '
2020-08-12 08:54:46 +00:00
errors_explained . append ( [ f ' <a href= " # { anchor } " >The query no. { row [ 3 ] } of test \' { row [ 2 ] } \' is taking too long to run. Keep the run time below { allowed_single_run_time } seconds " </a> ' ] )
2020-06-27 00:45:00 +00:00
slow_average_tests + = 1
else :
attrs [ 0 ] = ' '
2020-08-07 01:25:45 +00:00
text + = tableRow ( row , attrs , anchor )
2020-08-05 15:46:05 +00:00
text + = tableEnd ( )
tables . append ( text )
2020-06-27 00:45:00 +00:00
2020-08-05 15:46:05 +00:00
add_partial ( )
2020-06-27 00:45:00 +00:00
2020-08-05 15:46:05 +00:00
def add_changes ( ) :
2020-04-28 07:45:35 +00:00
rows = tsvRows ( ' report/changed-perf.tsv ' )
2020-04-02 18:44:58 +00:00
if not rows :
return
2020-08-05 15:46:05 +00:00
global faster_queries , slower_queries , tables
2020-04-02 18:44:58 +00:00
2020-09-17 16:21:59 +00:00
text = tableStart ( ' Changes in Performance ' )
2020-04-02 18:44:58 +00:00
columns = [
2020-06-25 20:19:27 +00:00
' Old, s ' , # 0
' New, s ' , # 1
2020-09-03 00:57:25 +00:00
' Ratio of speedup (-) or slowdown (+) ' , # 2
2020-07-30 22:13:50 +00:00
' Relative difference (new − old) / old ' , # 3
2020-09-10 15:48:39 +00:00
' p < 0.01 threshold ' , # 4
2020-09-23 20:45:39 +00:00
' ' , # Failed # 5
2020-07-30 22:13:50 +00:00
' Test ' , # 6
' # ' , # 7
' Query ' , # 8
2020-04-02 18:44:58 +00:00
]
attrs = [ ' ' for c in columns ]
2020-08-01 10:10:36 +00:00
attrs [ 5 ] = None
2020-09-23 20:45:39 +00:00
text + = tableHeader ( columns , attrs )
2020-04-02 18:44:58 +00:00
for row in rows :
2020-08-07 01:25:45 +00:00
anchor = f ' { currentTableAnchor ( ) } . { row [ 6 ] } . { row [ 7 ] } '
2020-07-30 22:13:50 +00:00
if int ( row [ 5 ] ) :
if float ( row [ 3 ] ) < 0. :
2020-04-21 18:46:45 +00:00
faster_queries + = 1
2020-07-30 22:13:50 +00:00
attrs [ 2 ] = attrs [ 3 ] = f ' style= " background: { color_good } " '
2020-04-21 18:46:45 +00:00
else :
slower_queries + = 1
2020-07-30 22:13:50 +00:00
attrs [ 2 ] = attrs [ 3 ] = f ' style= " background: { color_bad } " '
2020-08-07 01:25:45 +00:00
errors_explained . append ( [ f ' <a href= " # { anchor } " >The query no. { row [ 7 ] } of test \' { row [ 6 ] } \' has slowed down</a> ' ] )
2020-04-28 07:45:35 +00:00
else :
2020-07-30 22:13:50 +00:00
attrs [ 2 ] = attrs [ 3 ] = ' '
2020-04-02 18:44:58 +00:00
2020-08-07 01:25:45 +00:00
text + = tableRow ( row , attrs , anchor )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text + = tableEnd ( )
tables . append ( text )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_changes ( )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
def add_unstable_queries ( ) :
global unstable_queries , very_unstable_queries , tables
2020-04-02 18:44:58 +00:00
2020-04-28 07:45:35 +00:00
unstable_rows = tsvRows ( ' report/unstable-queries.tsv ' )
2020-04-02 18:44:58 +00:00
if not unstable_rows :
return
unstable_queries + = len ( unstable_rows )
columns = [
2020-06-25 20:19:27 +00:00
' Old, s ' , #0
' New, s ' , #1
2020-04-02 18:44:58 +00:00
' Relative difference (new - old)/old ' , #2
2020-09-10 15:48:39 +00:00
' p < 0.01 threshold ' , #3
2020-09-23 20:45:39 +00:00
' ' , # Failed #4
2020-04-28 07:45:35 +00:00
' Test ' , #5
2020-06-08 13:57:33 +00:00
' # ' , #6
' Query ' #7
2020-02-27 17:57:08 +00:00
]
2020-09-23 20:45:39 +00:00
attrs = [ ' ' for c in columns ]
attrs [ 4 ] = None
2020-02-27 17:57:08 +00:00
2020-09-17 16:21:59 +00:00
text = tableStart ( ' Unstable Queries ' )
2020-09-23 20:45:39 +00:00
text + = tableHeader ( columns , attrs )
2020-04-02 18:44:58 +00:00
for r in unstable_rows :
2020-08-07 01:25:45 +00:00
anchor = f ' { currentTableAnchor ( ) } . { r [ 5 ] } . { r [ 6 ] } '
2020-04-28 07:45:35 +00:00
if int ( r [ 4 ] ) :
2020-04-02 18:44:58 +00:00
very_unstable_queries + = 1
2020-05-20 02:19:19 +00:00
attrs [ 3 ] = f ' style= " background: { color_bad } " '
2020-04-02 18:44:58 +00:00
else :
attrs [ 3 ] = ' '
2021-05-13 09:16:43 +00:00
# Just don't add the slightly unstable queries we don't consider
# errors. It's not clear what the user should do with them.
continue
2020-04-02 18:44:58 +00:00
2020-08-07 01:25:45 +00:00
text + = tableRow ( r , attrs , anchor )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text + = tableEnd ( )
2021-05-26 13:30:43 +00:00
# Don't add an empty table.
if very_unstable_queries :
tables . append ( text )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_unstable_queries ( )
2020-04-02 18:44:58 +00:00
2020-05-20 02:19:19 +00:00
skipped_tests_rows = tsvRows ( ' analyze/skipped-tests.tsv ' )
2020-09-17 16:21:59 +00:00
addSimpleTable ( ' Skipped Tests ' , [ ' Test ' , ' Reason ' ] , skipped_tests_rows )
2020-04-02 18:44:58 +00:00
2020-09-17 16:21:59 +00:00
addSimpleTable ( ' Test Performance Changes ' ,
2020-09-03 00:57:25 +00:00
[ ' Test ' , ' Ratio of speedup (-) or slowdown (+) ' , ' Queries ' , ' Total not OK ' , ' Changed perf ' , ' Unstable ' ] ,
2020-06-09 13:29:07 +00:00
tsvRows ( ' report/test-perf-changes.tsv ' ) )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
def add_test_times ( ) :
global slow_average_tests , tables
2020-04-28 07:45:35 +00:00
rows = tsvRows ( ' report/test-times.tsv ' )
2020-04-02 18:44:58 +00:00
if not rows :
return
columns = [
2020-10-20 17:06:51 +00:00
' Test ' , #0
' Wall clock time, entire test, s ' , #1
' Total client time for measured query runs, s ' , #2
' Queries ' , #3
' Longest query, total for measured runs, s ' , #4
' Wall clock time per query, s ' , #5
' Shortest query, total for measured runs, s ' , #6
' ' , # Runs #7
2020-04-02 18:44:58 +00:00
]
2020-09-23 07:28:28 +00:00
attrs = [ ' ' for c in columns ]
attrs [ 7 ] = None
2020-04-02 18:44:58 +00:00
2020-09-17 16:21:59 +00:00
text = tableStart ( ' Test Times ' )
2020-09-23 20:45:39 +00:00
text + = tableHeader ( columns , attrs )
2020-05-09 15:58:47 +00:00
2020-09-23 08:21:55 +00:00
allowed_average_run_time = 1.6 # 30 seconds per test at 7 runs
2020-04-02 18:44:58 +00:00
for r in rows :
2020-09-01 19:05:57 +00:00
anchor = f ' { currentTableAnchor ( ) } . { r [ 0 ] } '
2020-09-18 15:06:41 +00:00
total_runs = ( int ( r [ 7 ] ) + 1 ) * 2 # one prewarm run, two servers
2020-09-30 12:03:52 +00:00
if r [ 0 ] != ' Total ' and float ( r [ 5 ] ) > allowed_average_run_time * total_runs :
2020-04-02 18:44:58 +00:00
# FIXME should be 15s max -- investigate parallel_insert
slow_average_tests + = 1
2020-09-03 01:42:25 +00:00
attrs [ 5 ] = f ' style= " background: { color_bad } " '
2020-09-01 19:05:57 +00:00
errors_explained . append ( [ f ' <a href= " # { anchor } " >The test \' { r [ 0 ] } \' is too slow to run as a whole. Investigate whether the create and fill queries can be sped up ' ] )
2020-04-02 18:44:58 +00:00
else :
2020-09-03 01:42:25 +00:00
attrs [ 5 ] = ' '
2020-04-02 18:44:58 +00:00
2020-09-30 12:03:52 +00:00
if r [ 0 ] != ' Total ' and float ( r [ 4 ] ) > allowed_single_run_time * total_runs :
2020-04-02 18:44:58 +00:00
slow_average_tests + = 1
2020-09-03 01:42:25 +00:00
attrs [ 4 ] = f ' style= " background: { color_bad } " '
2020-08-07 01:25:45 +00:00
errors_explained . append ( [ f ' <a href= " ./all-queries.html#all-query-times. { r [ 0 ] } .0 " >Some query of the test \' { r [ 0 ] } \' is too slow to run. See the all queries report ' ] )
2020-04-02 18:44:58 +00:00
else :
2020-09-03 01:42:25 +00:00
attrs [ 4 ] = ' '
2020-04-02 18:44:58 +00:00
2020-09-01 19:05:57 +00:00
text + = tableRow ( r , attrs , anchor )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text + = tableEnd ( )
tables . append ( text )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_test_times ( )
2020-04-02 18:44:58 +00:00
2020-09-17 16:21:59 +00:00
addSimpleTable ( ' Metric Changes ' ,
2020-06-25 20:19:27 +00:00
[ ' Metric ' , ' Old median value ' , ' New median value ' ,
' Relative difference ' , ' Times difference ' ] ,
tsvRows ( ' metrics/changes.tsv ' ) )
2020-08-05 15:46:05 +00:00
add_report_errors ( )
add_errors_explained ( )
for t in tables :
print ( t )
2020-04-02 18:44:58 +00:00
2021-04-23 13:47:13 +00:00
print ( f """
2020-08-28 22:46:04 +00:00
< / div >
2020-04-02 18:44:58 +00:00
< p class = " links " >
< a href = " all-queries.html " > All queries < / a >
< a href = " compare.log " > Log < / a >
2020-05-20 02:19:19 +00:00
< a href = " output.7z " > Test output < / a >
2021-04-23 13:47:13 +00:00
{ os . getenv ( " CHPC_ADD_REPORT_LINKS " ) or ' ' }
2020-04-02 18:44:58 +00:00
< / p >
< / body >
< / html >
""" )
status = ' success '
message = ' See the report '
message_array = [ ]
if slow_average_tests :
status = ' failure '
message_array . append ( str ( slow_average_tests ) + ' too long ' )
if faster_queries :
message_array . append ( str ( faster_queries ) + ' faster ' )
if slower_queries :
2020-05-13 03:55:19 +00:00
if slower_queries > 3 :
status = ' failure '
2020-04-02 18:44:58 +00:00
message_array . append ( str ( slower_queries ) + ' slower ' )
2020-06-27 00:45:00 +00:00
if unstable_partial_queries :
2021-05-26 13:30:43 +00:00
very_unstable_queries + = unstable_partial_queries
2020-06-27 00:45:00 +00:00
status = ' failure '
2021-05-12 22:32:53 +00:00
# Don't show mildly unstable queries, only the very unstable ones we
# treat as errors.
2021-05-12 09:47:08 +00:00
if very_unstable_queries :
2021-06-04 15:27:21 +00:00
if very_unstable_queries > 3 :
error_tests + = very_unstable_queries
status = ' failure '
2021-05-20 12:05:20 +00:00
message_array . append ( str ( very_unstable_queries ) + ' unstable ' )
2020-04-02 18:44:58 +00:00
error_tests + = slow_average_tests
if error_tests :
status = ' failure '
2020-06-10 21:38:26 +00:00
message_array . insert ( 0 , str ( error_tests ) + ' errors ' )
2020-04-02 18:44:58 +00:00
if message_array :
message = ' , ' . join ( message_array )
if report_errors :
status = ' failure '
message = ' Errors while building the report. '
2020-10-02 16:54:07 +00:00
print ( ( """
2020-04-02 18:44:58 +00:00
< ! - - status : { status } - - >
< ! - - message : { message } - - >
2020-10-02 16:54:07 +00:00
""" .format(status=status, message=message)))
2020-04-02 18:44:58 +00:00
elif args . report == ' all-queries ' :
2020-10-02 16:54:07 +00:00
print ( ( header_template . format ( ) ) )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_tested_commits ( )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
def add_all_queries ( ) :
2020-04-28 07:45:35 +00:00
rows = tsvRows ( ' report/all-queries.tsv ' )
2020-04-02 18:44:58 +00:00
if not rows :
return
columns = [
2020-09-23 20:45:39 +00:00
' ' , # Changed #0
' ' , # Unstable #1
2020-06-25 20:19:27 +00:00
' Old, s ' , #2
' New, s ' , #3
2020-09-03 00:57:25 +00:00
' Ratio of speedup (-) or slowdown (+) ' , #4
2020-07-30 22:13:50 +00:00
' Relative difference (new − old) / old ' , #5
2020-09-10 15:48:39 +00:00
' p < 0.01 threshold ' , #6
2020-04-28 07:45:35 +00:00
' Test ' , #7
2020-06-08 13:57:33 +00:00
' # ' , #8
' Query ' , #9
2020-04-02 18:44:58 +00:00
]
attrs = [ ' ' for c in columns ]
2020-04-28 07:45:35 +00:00
attrs [ 0 ] = None
attrs [ 1 ] = None
2020-09-23 20:45:39 +00:00
text = tableStart ( ' All Query Times ' )
text + = tableHeader ( columns , attrs )
2020-04-02 18:44:58 +00:00
for r in rows :
2020-08-07 01:25:45 +00:00
anchor = f ' { currentTableAnchor ( ) } . { r [ 7 ] } . { r [ 8 ] } '
2020-04-28 07:45:35 +00:00
if int ( r [ 1 ] ) :
2020-05-20 02:19:19 +00:00
attrs [ 6 ] = f ' style= " background: { color_bad } " '
2020-04-21 18:46:45 +00:00
else :
2020-04-28 07:45:35 +00:00
attrs [ 6 ] = ' '
2020-04-21 18:46:45 +00:00
2020-04-28 07:45:35 +00:00
if int ( r [ 0 ] ) :
2020-07-30 22:13:50 +00:00
if float ( r [ 5 ] ) > 0. :
attrs [ 4 ] = attrs [ 5 ] = f ' style= " background: { color_bad } " '
2020-04-21 18:46:45 +00:00
else :
2020-07-30 22:13:50 +00:00
attrs [ 4 ] = attrs [ 5 ] = f ' style= " background: { color_good } " '
2020-04-02 18:44:58 +00:00
else :
2020-07-30 22:13:50 +00:00
attrs [ 4 ] = attrs [ 5 ] = ' '
2020-04-02 18:44:58 +00:00
2020-04-29 17:26:28 +00:00
if ( float ( r [ 2 ] ) + float ( r [ 3 ] ) ) / 2 > allowed_single_run_time :
2020-05-20 02:19:19 +00:00
attrs [ 2 ] = f ' style= " background: { color_bad } " '
attrs [ 3 ] = f ' style= " background: { color_bad } " '
2020-04-29 17:26:28 +00:00
else :
attrs [ 2 ] = ' '
attrs [ 3 ] = ' '
2020-08-07 01:25:45 +00:00
text + = tableRow ( r , attrs , anchor )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
text + = tableEnd ( )
tables . append ( text )
2020-04-02 18:44:58 +00:00
2020-08-05 15:46:05 +00:00
add_all_queries ( )
add_report_errors ( )
for t in tables :
print ( t )
2020-04-30 08:36:33 +00:00
2021-04-23 13:47:13 +00:00
print ( f """
2020-08-28 22:46:04 +00:00
< / div >
2020-04-02 18:44:58 +00:00
< p class = " links " >
< a href = " report.html " > Main report < / a >
< a href = " compare.log " > Log < / a >
2020-05-20 02:19:19 +00:00
< a href = " output.7z " > Test output < / a >
2021-04-23 13:47:13 +00:00
{ os . getenv ( " CHPC_ADD_REPORT_LINKS " ) or ' ' }
2020-04-02 18:44:58 +00:00
< / p >
< / body >
< / html >
""" )
