From e59aaa88f8536e1606cc15cda7c15c93834c202f Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com>
Date: Fri, 17 Apr 2020 16:32:59 +0300
Subject: [PATCH] Update compare.sh

---
 docker/test/performance-comparison/compare.sh | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh
index bf48fe467ca..7e9da2b798b 100755
--- a/docker/test/performance-comparison/compare.sh
+++ b/docker/test/performance-comparison/compare.sh
@@ -230,12 +230,14 @@ create table queries engine File(TSVWithNamesAndTypes, 'queries.rep')
         -- but the right way to do this is not yet clear.
         left + right < 0.05 as short,
 
-        not short and abs(diff) < 0.10 and rd[3] > 0.10 as unstable,
-
-        -- Do not consider changed the queries with 5% RD below 5% -- e.g., we're
-        -- likely to observe a difference > 5% in less than 5% cases.
-        -- Not sure it is correct, but empirically it filters out a lot of noise.
-        not short and abs(diff) > 0.15 and abs(diff) > rd[3] and rd[1] > 0.05 as changed,
+        -- Difference > 15% and > rd(99%) -- changed. We can't filter out flaky
+        -- queries by rd(5%), because it can be zero when the difference is smaller
+        -- than a typical distribution width. The difference is still real though.
+        not short and abs(diff) > 0.15 and abs(diff) > rd[4] as changed,
+        
+        -- Not changed but rd(99%) > 10% -- unstable.
+        not short and not changed and rd[4] > 0.10 as unstable,
+        
         left, right, diff, rd,
         replaceAll(_file, '-report.tsv', '') test,
         query