From 0c6838a04377f6373aa341414e46c6559be953ac Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Dec 2020 11:48:40 +0300 Subject: [PATCH 1/2] Fix glaky 01076_parallel_alter test --- .../01076_parallel_alter_replicated_zookeeper.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh index 4f86d7c1666..c9e5d7b9447 100755 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -103,8 +103,10 @@ done sleep 1 counter=0 +have_undone_mutations_query="select * from system.mutations where table like 'concurrent_mutate_mt_%' and is_done=0 and database='${CLICKHOUSE_DATABASE}'" +have_all_tables_query="select count() FROM system.tables WHERE name LIKE 'concurrent_mutate_mt_%' and database='${CLICKHOUSE_DATABASE}'" -while [[ $($CLICKHOUSE_CLIENT --query "select * from system.mutations where table like 'concurrent_mutate_mt_%' and is_done=0" 2>&1) ]]; do +while true ; do if [ "$counter" -gt 120 ] then break @@ -113,7 +115,13 @@ while [[ $($CLICKHOUSE_CLIENT --query "select * from system.mutations where tabl for i in $(seq $REPLICAS); do $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_mutate_mt_$i" 2> /dev/null done + counter=$(($counter + 1)) + + # no active mutations and all tables attached + if [[ -z $($CLICKHOUSE_CLIENT --query "$have_undone_mutations_query" 2>&1) && $($CLICKHOUSE_CLIENT --query "$have_all_tables_query" 2>&1) == "$REPLICAS" ]]; then + break + fi done for i in $(seq $REPLICAS); do From ce2028e3ceab6e601b40006df7ae1101acee20c3 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Dec 2020 14:19:12 +0300 Subject: [PATCH 2/2] Fix clickhouse-test --- tests/clickhouse-test | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 45cde00037f..07677d5b823 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -251,7 +251,7 @@ stop_time = None # def run_tests_array(all_tests, suite, suite_dir, suite_tmp_dir, run_total): def run_tests_array(all_tests_with_params): - all_tests, suite, suite_dir, suite_tmp_dir, run_total = all_tests_with_params + all_tests, suite, suite_dir, suite_tmp_dir = all_tests_with_params global exit_code global SERVER_DIED global stop_time @@ -698,6 +698,7 @@ def main(args): all_tests = [t for t in all_tests if any([re.search(r, t) for r in args.test])] all_tests.sort(key=key_func) + jobs = args.jobs parallel_tests = [] sequential_tests = [] for test in all_tests: @@ -706,35 +707,32 @@ def main(args): else: parallel_tests.append(test) - print("Found", len(parallel_tests), "parallel tests and", len(sequential_tests), "sequential tests") - run_n, run_total = args.parallel.split('/') - run_n = float(run_n) - run_total = float(run_total) - tests_n = len(parallel_tests) - if run_total > tests_n: - run_total = tests_n - if run_n > run_total: - continue + if jobs > 1 and len(parallel_tests) > 0: + print("Found", len(parallel_tests), "parallel tests and", len(sequential_tests), "sequential tests") + run_n, run_total = args.parallel.split('/') + run_n = float(run_n) + run_total = float(run_total) + tests_n = len(parallel_tests) + if run_total > tests_n: + run_total = tests_n - jobs = args.jobs - if jobs > tests_n: - jobs = tests_n - if jobs > run_total: - run_total = jobs + if jobs > tests_n: + jobs = tests_n + if jobs > run_total: + run_total = jobs - batch_size = len(parallel_tests) // jobs - parallel_tests_array = [] - for i in range(0, len(parallel_tests), batch_size): - parallel_tests_array.append((parallel_tests[i:i+batch_size], suite, suite_dir, suite_tmp_dir, run_total)) + batch_size = len(parallel_tests) // jobs + parallel_tests_array = [] + for i in range(0, len(parallel_tests), batch_size): + parallel_tests_array.append((parallel_tests[i:i+batch_size], suite, suite_dir, suite_tmp_dir)) - if jobs > 1: with closing(multiprocessing.Pool(processes=jobs)) as pool: pool.map(run_tests_array, parallel_tests_array) - run_tests_array((sequential_tests, suite, suite_dir, suite_tmp_dir, run_total)) + run_tests_array((sequential_tests, suite, suite_dir, suite_tmp_dir)) total_tests_run += len(sequential_tests) + len(parallel_tests) else: - run_tests_array((all_tests, suite, suite_dir, suite_tmp_dir, run_total)) + run_tests_array((all_tests, suite, suite_dir, suite_tmp_dir)) total_tests_run += len(all_tests) if args.hung_check: