fixup

docker/test/performance-comparison/compare.sh

@@ -421,9 +421,39 @@ create table queries_for_flamegraph engine File(TSVWithNamesAndTypes,
     select test, query_index from queries where unstable_show or changed_show
     ;
 
-create table unstable_tests_tsv engine File(TSV, 'report/bad-tests.tsv') as
+create table test_time_changes_tsv engine File(TSV, 'report/test-time-changes.tsv') as
-    select test, sum(unstable_fail) u, sum(changed_fail) c, u + c s from queries
+    select test, queries, average_time_change from (
-    group by test having s > 0 order by s desc;
+        select test, count(*) queries,
+            sum(left) as left, sum(right) as right,
+            (right - left) / right average_time_change
+        from queries
+        group by test
+        order by abs(average_time_change) desc
+    )
+    ;
+
+create table unstable_tests_tsv engine File(TSV, 'report/unstable-tests.tsv') as
+    select test, sum(unstable_show) total_unstable, sum(changed_show) total_changed
+    from queries
+    group by test
+    order by total_unstable + total_changed desc
+    ;
+
+create table test_perf_changes_tsv engine File(TSV, 'report/test-perf-changes.tsv') as
+    select test,
+        queries,
+        coalesce(total_unstable, 0) total_unstable,
+        coalesce(total_changed, 0) total_changed,
+        total_unstable + total_changed total_bad,
+        coalesce(toString(floor(average_time_change, 3)), '??') average_time_change_str
+    from test_time_changes_tsv
+    full join unstable_tests_tsv
+    using test
+    where (abs(average_time_change) > 0.05 and queries > 5)
+        or (total_bad > 0)
+    order by total_bad desc, average_time_change desc
+    settings join_use_nulls = 1
+    ;
 
 create table query_time engine Memory as select *
     from file('analyze/client-times.tsv', TSV,

docker/test/performance-comparison/report.py

@@ -274,9 +274,9 @@ if args.report == 'main':
     skipped_tests_rows = tsvRows('analyze/skipped-tests.tsv')
     printSimpleTable('Skipped tests', ['Test', 'Reason'], skipped_tests_rows)
 
-    printSimpleTable('Tests with most unstable queries',
+    printSimpleTable('Test performance changes',
-        ['Test', 'Unstable', 'Changed perf', 'Total not OK'],
+        ['Test', 'Queries', 'Unstable', 'Changed perf', 'Total not OK', 'Avg relative time diff'],
-        tsvRows('report/bad-tests.tsv'))
+        tsvRows('report/test-perf-changes.tsv'))
 
     def print_test_times():
         global slow_average_tests

tests/performance/base64_hits.xml

@@ -1,6 +1,6 @@
 <test>
     <preconditions>
-        <table_exists>hits_100m_single</table_exists>
+        <table_exists>hits_10m_single</table_exists>
     </preconditions>
 
     <settings>
@@ -19,7 +19,7 @@
         </substitution>
     </substitutions>
 
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(base64Encode({string}))</query>
+    <query>SELECT count() FROM hits_10m_single WHERE NOT ignore(base64Encode({string}))</query>
-    <query>SELECT count() FROM hits_100m_single WHERE base64Decode(base64Encode({string})) != {string}</query>
+    <query>SELECT count() FROM hits_10m_single WHERE base64Decode(base64Encode({string})) != {string}</query>
-    <query>SELECT count() FROM hits_100m_single WHERE tryBase64Decode(base64Encode({string})) != {string}</query>
+    <query>SELECT count() FROM hits_10m_single WHERE tryBase64Decode(base64Encode({string})) != {string}</query>
 </test>

tests/performance/basename.xml

@@ -1,12 +1,12 @@
 <test>
     <preconditions>
-        <table_exists>hits_100m_single</table_exists>
+        <table_exists>hits_10m_single</table_exists>
     </preconditions>
 
     <settings>
         <max_threads>1</max_threads>
     </settings>
 
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(basename(URL))</query>
+    <query>SELECT count() FROM hits_10m_single WHERE NOT ignore(basename(URL))</query>
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(basename(Referer))</query>
+    <query>SELECT count() FROM hits_10m_single WHERE NOT ignore(basename(Referer))</query>
 </test>

tests/performance/int_parsing.xml

@@ -14,7 +14,6 @@
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(OS)) SETTINGS max_threads = 1</query>
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(UserAgent)) SETTINGS max_threads = 1</query>
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(Refresh)) SETTINGS max_threads = 1</query>
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(IsRobot)) SETTINGS max_threads = 1</query>
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResolutionWidth)) SETTINGS max_threads = 1</query>
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResolutionHeight)) SETTINGS max_threads = 1</query>
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResolutionDepth)) SETTINGS max_threads = 1</query>
@@ -67,15 +66,6 @@
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResponseStartTiming)) SETTINGS max_threads = 1</query>
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResponseEndTiming)) SETTINGS max_threads = 1</query>
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(FetchTiming)) SETTINGS max_threads = 1</query>
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RedirectTiming)) SETTINGS max_threads = 1</query>
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(DOMInteractiveTiming)) SETTINGS max_threads = 1</query>
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(DOMContentLoadedTiming)) SETTINGS max_threads = 1</query>
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(DOMCompleteTiming)) SETTINGS max_threads = 1</query>
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(LoadEventStartTiming)) SETTINGS max_threads = 1</query>
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(LoadEventEndTiming)) SETTINGS max_threads = 1</query>
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(NSToDOMContentLoadedTiming)) SETTINGS max_threads = 1</query>
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(FirstPaintTiming)) SETTINGS max_threads = 1</query>
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RedirectCount)) SETTINGS max_threads = 1</query>
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(SocialSourceNetworkID)) SETTINGS max_threads = 1</query>
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ParamPrice)) SETTINGS max_threads = 1</query>
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ParamCurrencyID)) SETTINGS max_threads = 1</query>
@@ -83,7 +73,4 @@
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RefererHash)) SETTINGS max_threads = 1</query>
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(URLHash)) SETTINGS max_threads = 1</query>
     <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(CLID)) SETTINGS max_threads = 1</query>
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(YCLID)) SETTINGS max_threads = 1</query>
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RequestNum)) SETTINGS max_threads = 1</query>
-    <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RequestTry)) SETTINGS max_threads = 1</query>
 </test>