Merge pull request #14430 from ClickHouse/aku/perf4

[wip] perf experiment no.4 -- geomean

docker/test/performance-comparison/compare.sh

@@ -565,40 +565,54 @@ create table unstable_queries_report engine File(TSV, 'report/unstable-queries.t
         toDecimal64(stat_threshold, 3), unstable_fail, test, query_index, query_display_name
     from queries where unstable_show order by stat_threshold desc;
 
-create table test_time_changes engine File(TSV, 'report/test-time-changes.tsv') as
-    select test, queries, average_time_change from (
-        select test, count(*) queries,
-            sum(left) as left, sum(right) as right,
-            (right - left) / right average_time_change
-        from queries
-        group by test
-        order by abs(average_time_change) desc
-    )
-    ;
 
-create table unstable_tests engine File(TSV, 'report/unstable-tests.tsv') as
+create view test_speedup as
-    select test, sum(unstable_show) total_unstable, sum(changed_show) total_changed
+    select
+        test,
+        exp2(avg(log2(left / right))) times_speedup,
+        count(*) queries,
+        unstable + changed bad,
+        sum(changed_show) changed,
+        sum(unstable_show) unstable
     from queries
     group by test
-    order by total_unstable + total_changed desc
+    order by times_speedup desc
+    ;
+
+create view total_speedup as
+    select
+        'Total' test,
+        exp2(avg(log2(times_speedup))) times_speedup,
+        sum(queries) queries,
+        unstable + changed bad,
+        sum(changed) changed,
+        sum(unstable) unstable
+    from test_speedup
     ;
 
 create table test_perf_changes_report engine File(TSV, 'report/test-perf-changes.tsv') as
-    select test,
+    with
-        queries,
+        (times_speedup >= 1
-        coalesce(total_unstable, 0) total_unstable,
+            ? '-' || toString(toDecimal64(times_speedup, 3)) || 'x'
-        coalesce(total_changed, 0) total_changed,
+            : '+' || toString(toDecimal64(1 / times_speedup, 3)) || 'x')
-        total_unstable + total_changed total_bad,
+        as times_speedup_str
-        coalesce(toString(toDecimal64(average_time_change, 3)), '??') average_time_change_str
+    select test, times_speedup_str, queries, bad, changed, unstable
-    from test_time_changes
+    -- Not sure what's the precedence of UNION ALL vs WHERE & ORDER BY, hence all
-    full join unstable_tests
+    -- the braces.
-    using test
+    from (
-    where (abs(average_time_change) > 0.05 and queries > 5)
+        (
-        or (total_bad > 0)
+            select * from total_speedup
-    order by total_bad desc, average_time_change desc
+        ) union all (
-    settings join_use_nulls = 1
+            select * from test_speedup
+            where
+                (times_speedup >= 1 ? times_speedup : (1 / times_speedup)) >= 1.005
+                or bad
+        )
+    )
+    order by test = 'Total' desc
     ;
 
+
 create view total_client_time_per_query as select *
     from file('analyze/client-times.tsv', TSV,
         'test text, query_index int, client float, server float');

docker/test/performance-comparison/report.py

@@ -370,7 +370,7 @@ if args.report == 'main':
         columns = [
             'Old, s',                                          # 0
             'New, s',                                          # 1
-            'Times speedup / slowdown',                 # 2
+            'Ratio of speedup (-) or slowdown (+)',                 # 2
             'Relative difference (new − old) / old',   # 3
             'p&nbsp;<&nbsp;0.001 threshold',                   # 4
             # Failed                                           # 5
@@ -447,7 +447,7 @@ if args.report == 'main':
     addSimpleTable('Skipped tests', ['Test', 'Reason'], skipped_tests_rows)
 
     addSimpleTable('Test performance changes',
-        ['Test', 'Queries', 'Unstable', 'Changed perf', 'Total not OK', 'Avg relative time diff'],
+        ['Test', 'Ratio of speedup&nbsp;(-) or slowdown&nbsp;(+)', 'Queries', 'Total not OK', 'Changed perf', 'Unstable'],
         tsvRows('report/test-perf-changes.tsv'))
 
     def add_test_times():
@@ -647,7 +647,7 @@ elif args.report == 'all-queries':
             # Unstable #1
             'Old,&nbsp;s', #2
             'New,&nbsp;s', #3
-            'Times speedup / slowdown',                 #4
+            'Ratio of speedup&nbsp;(-) or slowdown&nbsp;(+)',                 #4
             'Relative difference (new&nbsp;&minus;&nbsp;old) / old', #5
             'p&nbsp;&lt;&nbsp;0.001 threshold',          #6
             'Test',                                   #7