diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 12ee436e400..bf7549a83c4 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -125,13 +125,13 @@ def clear_ip_tables_and_restart_daemons(): logging.info("Dump iptables after run %s", subprocess.check_output("iptables -L", shell=True)) try: logging.info("Killing all alive docker containers") - subprocess.check_output("docker kill $(docker ps -q)", shell=True) + subprocess.check_output("timeout -s 9 10m docker kill $(docker ps -q)", shell=True) except subprocess.CalledProcessError as err: logging.info("docker kill excepted: " + str(err)) try: logging.info("Removing all docker containers") - subprocess.check_output("docker rm $(docker ps -a -q) --force", shell=True) + subprocess.check_output("timeout -s 9 10m docker rm $(docker ps -a -q) --force", shell=True) except subprocess.CalledProcessError as err: logging.info("docker rm excepted: " + str(err)) @@ -376,6 +376,24 @@ class ClickhouseIntegrationTestsRunner: res.add(path) return res + def try_run_test_group(self, repo_path, test_group, tests_in_group, num_tries, num_workers): + try: + return self.run_test_group(repo_path, test_group, tests_in_group, num_tries, num_workers) + except Exception as e: + logging.info("Failed to run {}:\n{}".format(str(test_group), str(e))) + counters = { + "ERROR": [], + "PASSED": [], + "FAILED": [], + "SKIPPED": [], + "FLAKY": [], + } + tests_times = defaultdict(float) + for test in tests_in_group: + counters["ERROR"].append(test) + tests_times[test] = 0 + return counters, tests_times, [] + def run_test_group(self, repo_path, test_group, tests_in_group, num_tries, num_workers): counters = { "ERROR": [], @@ -507,7 +525,7 @@ class ClickhouseIntegrationTestsRunner: for i in range(TRIES_COUNT): final_retry += 1 logging.info("Running tests for the %s time", i) - counters, tests_times, log_paths = self.run_test_group(repo_path, "flaky", tests_to_run, 1, 1) + counters, tests_times, log_paths = self.try_run_test_group(repo_path, "flaky", tests_to_run, 1, 1) logs += log_paths if counters["FAILED"]: logging.info("Found failed tests: %s", ' '.join(counters["FAILED"])) @@ -583,7 +601,7 @@ class ClickhouseIntegrationTestsRunner: for group, tests in items_to_run: logging.info("Running test group %s countaining %s tests", group, len(tests)) - group_counters, group_test_times, log_paths = self.run_test_group(repo_path, group, tests, MAX_RETRY, NUM_WORKERS) + group_counters, group_test_times, log_paths = self.try_run_test_group(repo_path, group, tests, MAX_RETRY, NUM_WORKERS) total_tests = 0 for counter, value in group_counters.items(): logging.info("Tests from group %s stats, %s count %s", group, counter, len(value))