From 8070562bd00e61ab64f3a41bd1b8e7383e729da4 Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <akuzm@yandex-team.ru>
Date: Wed, 27 Jan 2021 18:24:39 +0300
Subject: [PATCH] Run tests in parallel in flaky check

---
 docker/test/stateless/run.sh | 11 +++++------
 tests/clickhouse-test        |  7 +++++--
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index d9a03f84726..fb510a87fcd 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -55,12 +55,11 @@ function run_tests()
         ADDITIONAL_OPTIONS+=('00000_no_tests_to_skip')
     fi
 
-    for _ in $(seq 1 "$NUM_TRIES"); do
-        clickhouse-test --testname --shard --zookeeper --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a test_output/test_result.txt
-        if [ "${PIPESTATUS[0]}" -ne "0" ]; then
-            break;
-        fi
-    done
+    clickhouse-test --testname --shard --zookeeper --hung-check --print-time \
+            --test-runs "$NUM_TRIES" --jobs 4 \
+            "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
+        | ts '%Y-%m-%d %H:%M:%S' \
+        | tee -a test_output/test_result.txt
 }
 
 export -f run_tests
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index d5c6019d28f..70f7fe18982 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -342,9 +342,10 @@ def run_tests_array(all_tests_with_params):
                             SERVER_DIED = True
                             break
 
+                    file_suffix = ('.' + str(os.getpid())) if is_concurrent else ''
                     reference_file = os.path.join(suite_dir, name) + '.reference'
-                    stdout_file = os.path.join(suite_tmp_dir, name) + '.stdout'
-                    stderr_file = os.path.join(suite_tmp_dir, name) + '.stderr'
+                    stdout_file = os.path.join(suite_tmp_dir, name) + file_suffix + '.stdout'
+                    stderr_file = os.path.join(suite_tmp_dir, name) + file_suffix + '.stderr'
 
                     proc, stdout, stderr, total_time = run_single_test(args, ext, server_logs_level, client_options, case_file, stdout_file, stderr_file)
 
@@ -717,6 +718,7 @@ def main(args):
             all_tests = [case for case in all_tests if is_test_from_dir(suite_dir, case)]
             if args.test:
                 all_tests = [t for t in all_tests if any([re.search(r, t) for r in args.test])]
+            all_tests = all_tests * args.test_runs
             all_tests.sort(key=key_func)
 
             jobs = args.jobs
@@ -886,6 +888,7 @@ if __name__ == '__main__':
     parser.add_argument('--database', help='Database for tests (random name test_XXXXXX by default)')
     parser.add_argument('--parallel', default='1/1', help='One parallel test run number/total')
     parser.add_argument('-j', '--jobs', default=1, nargs='?', type=int, help='Run all tests in parallel')
+    parser.add_argument('--test-runs', default=1, nargs='?', type=int, help='Run each test many times (useful for e.g. flaky check)')
     parser.add_argument('-U', '--unified', default=3, type=int, help='output NUM lines of unified context')
     parser.add_argument('-r', '--server-check-retries', default=30, type=int, help='Num of tries to execute SELECT 1 before tests started')
     parser.add_argument('--skip-list-path', help="Path to skip-list file")