From 313b576aec51d0cb54d29c8f8e066fa92dc76a14 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 24 Apr 2021 15:17:38 +0300
Subject: [PATCH] Improve retry logic in
 01675_distributed_bytes_to_delay_insert_long

https://clickhouse-test-reports.s3.yandex.net/23517/414d8d0634ea5e3f05d30f53df8531da54a7ff50/functional_stateless_tests_(ubsan).html#fail1
---
 ..._distributed_bytes_to_delay_insert_long.sh | 47 ++++++++++---------
 1 file changed, 24 insertions(+), 23 deletions(-)

diff --git a/tests/queries/0_stateless/01675_distributed_bytes_to_delay_insert_long.sh b/tests/queries/0_stateless/01675_distributed_bytes_to_delay_insert_long.sh
index 9c1d1bbe838..5687fe323d0 100755
--- a/tests/queries/0_stateless/01675_distributed_bytes_to_delay_insert_long.sh
+++ b/tests/queries/0_stateless/01675_distributed_bytes_to_delay_insert_long.sh
@@ -51,8 +51,8 @@ function test_max_delay_to_insert_will_throw()
 #
 function test_max_delay_to_insert_will_succeed_once()
 {
-    local flush_delay=$1 && shift
-    local max_delay_to_insert=$1 && shift
+    local max_delay_to_insert=4
+    local flush_delay=2
 
     drop_tables
 
@@ -64,50 +64,51 @@ function test_max_delay_to_insert_will_succeed_once()
 
     function flush_distributed_worker()
     {
-        sleep "$flush_delay"
+        sleep $flush_delay
         ${CLICKHOUSE_CLIENT} -q "system flush distributed dist_01675"
     }
     flush_distributed_worker &
 
     local start_seconds=$SECONDS
+    # ignore stderr, since it may produce exception if flushing thread will be too slow
+    # (this is possible on CI)
     ${CLICKHOUSE_CLIENT} --testmode -nq "
         -- first batch is always OK, since there is no pending bytes yet
         insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0;
         -- second will succeed, due to SYSTEM FLUSH DISTRIBUTED in background.
         insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0;
-    "
+    " >& /dev/null
     local end_seconds=$SECONDS
     wait
 
-    echo $((end_seconds-start_seconds))
+    local diff=$(( end_seconds-start_seconds ))
+
+    if (( diff<(flush_delay-1) )); then
+        # this is fatal error, that should not be retriable
+        echo "max_delay_to_insert was not wait flush_delay ($diff)"
+        exit 1
+    fi
+
+    # retry the test until the diff will be satisfied
+    # (since we cannot assume that there will be no other lags)
+    if (( diff>=(max_delay_to_insert-1) )); then
+        return 1
+    fi
+
+    return 0
 }
 function test_max_delay_to_insert_will_succeed()
 {
     echo "max_delay_to_insert will succeed"
 
-    local max_delay_to_insert=4
-    local flush_delay=2
-    local diff
     local retries=20 i=0
-
     while (( (i++) < retries )); do
-        diff=$(test_max_delay_to_insert_will_succeed_once $flush_delay $max_delay_to_insert)
-
-        if (( diff<(flush_delay-1) )); then
-            echo "max_delay_to_insert was not wait flush_delay ($diff)"
-            break
+        if test_max_delay_to_insert_will_succeed_once; then
+            return
         fi
-
-        # retry the test until the diff will be satisfied
-        # (since we cannot assume that there will be no other lags)
-        if (( diff>=(max_delay_to_insert-1) )); then
-            continue
-        fi
-
-        return
     done
 
-    echo "max_delay_to_insert was overcommited ($diff)"
+    echo failed
 }
 
 function run_test()