Merge pull request #66986 from qoega/integration-flaky-check-repeat

Add hardening to integration tests flaky check: repeat test case multiple times with same environment.

tests/ci/integration_tests_runner.py

@@ -29,7 +29,8 @@ CLICKHOUSE_BINARY_PATH = "usr/bin/clickhouse"
 CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH = "usr/bin/clickhouse-odbc-bridge"
 CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH = "usr/bin/clickhouse-library-bridge"
 
-FLAKY_TRIES_COUNT = 10
+FLAKY_TRIES_COUNT = 10  # run whole pytest several times
+FLAKY_REPEAT_COUNT = 5  # runs test case in single module several times
 MAX_TIME_SECONDS = 3600
 
 MAX_TIME_IN_SANDBOX = 20 * 60  # 20 minutes
@@ -568,6 +569,7 @@ class ClickhouseIntegrationTestsRunner:
         tests_in_group,
         num_tries,
         num_workers,
+        repeat_count,
     ):
         try:
             return self.run_test_group(
@@ -576,6 +578,7 @@ class ClickhouseIntegrationTestsRunner:
                 tests_in_group,
                 num_tries,
                 num_workers,
+                repeat_count,
             )
         except Exception as e:
             logging.info("Failed to run %s:\n%s", test_group, e)
@@ -598,6 +601,7 @@ class ClickhouseIntegrationTestsRunner:
         tests_in_group,
         num_tries,
         num_workers,
+        repeat_count,
     ):
         counters = {
             "ERROR": [],
@@ -639,6 +643,7 @@ class ClickhouseIntegrationTestsRunner:
 
             test_cmd = " ".join([shlex.quote(test) for test in sorted(test_names)])
             parallel_cmd = f" --parallel {num_workers} " if num_workers > 0 else ""
+            repeat_cmd = f" --count {repeat_count} " if repeat_count > 0 else ""
             # -r -- show extra test summary:
             # -f -- (f)ailed
             # -E -- (E)rror
@@ -647,7 +652,7 @@ class ClickhouseIntegrationTestsRunner:
             cmd = (
                 f"cd {repo_path}/tests/integration && "
                 f"timeout --signal=KILL 1h ./runner {self._get_runner_opts()} "
-                f"{image_cmd} -t {test_cmd} {parallel_cmd} -- -rfEps --run-id={i} "
+                f"{image_cmd} -t {test_cmd} {parallel_cmd} {repeat_cmd} -- -rfEps --run-id={i} "
                 f"--color=no --durations=0 {_get_deselect_option(self.should_skip_tests())} "
                 f"| tee {info_path}"
             )
@@ -784,7 +789,12 @@ class ClickhouseIntegrationTestsRunner:
             final_retry += 1
             logging.info("Running tests for the %s time", i)
             counters, tests_times, log_paths = self.try_run_test_group(
-                repo_path, "bugfix" if should_fail else "flaky", tests_to_run, 1, 1
+                repo_path,
+                "bugfix" if should_fail else "flaky",
+                tests_to_run,
+                1,
+                1,
+                FLAKY_REPEAT_COUNT,
             )
             logs += log_paths
             if counters["FAILED"]:
@@ -919,7 +929,7 @@ class ClickhouseIntegrationTestsRunner:
         for group, tests in items_to_run:
             logging.info("Running test group %s containing %s tests", group, len(tests))
             group_counters, group_test_times, log_paths = self.try_run_test_group(
-                repo_path, group, tests, MAX_RETRY, NUM_WORKERS
+                repo_path, group, tests, MAX_RETRY, NUM_WORKERS, 0
             )
             total_tests = 0
             for counter, value in group_counters.items():

tests/integration/runner

@@ -243,6 +243,10 @@ if __name__ == "__main__":
         "-n", "--parallel", action="store", dest="parallel", help="Parallelism"
     )
 
+    parser.add_argument(
+        "--count", action="store", type=int, dest="count", help="Repeat count"
+    )
+
     parser.add_argument(
         "--no-random",
         action="store",
@@ -318,6 +322,10 @@ if __name__ == "__main__":
         parallel_args += "--dist=loadfile"
         parallel_args += f" -n {args.parallel}".format()
 
+    repeat_args = (
+        f" --count {args.count}" if args.count is not None and args.count > 0 else ""
+    )
+
     rand_args = ""
     # if not args.no_random:
     #     rand_args += f"--random-seed={os.getpid()}"
@@ -409,7 +417,7 @@ if __name__ == "__main__":
         f"--volume={args.utils_dir}/grpc-client/pb2:/ClickHouse/utils/grpc-client/pb2 "
         f"--volume=/run:/run/host:ro {dockerd_internal_volume} {env_tags} {env_cleanup} "
         f"-e DOCKER_CLIENT_TIMEOUT=300 -e COMPOSE_HTTP_TIMEOUT=600 {use_old_analyzer} -e PYTHONUNBUFFERED=1 "
-        f'-e PYTEST_ADDOPTS="{parallel_args} {pytest_opts} {tests_list} {rand_args} -vvv"'
+        f'-e PYTEST_ADDOPTS="{parallel_args} {repeat_args} {pytest_opts} {tests_list} {rand_args} -vvv"'
         f" {DIND_INTEGRATION_TESTS_IMAGE_NAME}:{args.docker_image_version}"
     )
 

tests/integration/test_backup_restore_new/test.py

@@ -1,9 +1,10 @@
-import pytest
 import glob
-import re
-import random
 import os.path
+import pytest
+import random
+import re
 import sys
+import uuid
 from collections import namedtuple
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import assert_eq_with_retry, TSV
@@ -538,7 +539,8 @@ def test_backup_not_found_or_already_exists():
 
 
 def test_file_engine():
-    backup_name = f"File('/backups/file/')"
+    id = uuid.uuid4()
+    backup_name = f"File('/backups/file/{id}/')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -549,6 +551,7 @@ def test_file_engine():
 
     instance.query(f"RESTORE TABLE test.table FROM {backup_name}")
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
+    instance.query("DROP TABLE test.table")
 
 
 def test_database():
@@ -565,7 +568,8 @@ def test_database():
 
 
 def test_zip_archive():
-    backup_name = f"Disk('backups', 'archive.zip')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_{id}.zip')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -578,10 +582,12 @@ def test_zip_archive():
 
     instance.query(f"RESTORE TABLE test.table FROM {backup_name}")
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
+    instance.query("DROP TABLE test.table")
 
 
 def test_zip_archive_with_settings():
-    backup_name = f"Disk('backups', 'archive_with_settings.zip')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_with_settings_{id}.zip')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -596,10 +602,12 @@ def test_zip_archive_with_settings():
         f"RESTORE TABLE test.table FROM {backup_name} SETTINGS password='qwerty'"
     )
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
+    instance.query("DROP TABLE test.table")
 
 
 def test_zip_archive_with_bad_compression_method():
-    backup_name = f"Disk('backups', 'archive_with_bad_compression_method.zip')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_with_bad_compression_method_{id}.zip')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -617,7 +625,8 @@ def test_zip_archive_with_bad_compression_method():
 
 
 def test_tar_archive():
-    backup_name = f"Disk('backups', 'archive.tar')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_{id}.tar')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -630,10 +639,12 @@ def test_tar_archive():
 
     instance.query(f"RESTORE TABLE test.table FROM {backup_name}")
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
+    instance.query("DROP TABLE test.table")
 
 
 def test_tar_bz2_archive():
-    backup_name = f"Disk('backups', 'archive.tar.bz2')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_{id}.tar.bz2')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -649,7 +660,8 @@ def test_tar_bz2_archive():
 
 
 def test_tar_gz_archive():
-    backup_name = f"Disk('backups', 'archive.tar.gz')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_{id}.tar.gz')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -665,7 +677,8 @@ def test_tar_gz_archive():
 
 
 def test_tar_lzma_archive():
-    backup_name = f"Disk('backups', 'archive.tar.lzma')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_{id}.tar.lzma')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -681,7 +694,8 @@ def test_tar_lzma_archive():
 
 
 def test_tar_zst_archive():
-    backup_name = f"Disk('backups', 'archive.tar.zst')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_{id}.tar.zst')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -697,7 +711,8 @@ def test_tar_zst_archive():
 
 
 def test_tar_xz_archive():
-    backup_name = f"Disk('backups', 'archive.tar.xz')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_{id}.tar.xz')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -713,7 +728,8 @@ def test_tar_xz_archive():
 
 
 def test_tar_archive_with_password():
-    backup_name = f"Disk('backups', 'archive_with_password.tar')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_with_password_{id}.tar')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -731,7 +747,8 @@ def test_tar_archive_with_password():
 
 
 def test_tar_archive_with_bad_compression_method():
-    backup_name = f"Disk('backups', 'archive_with_bad_compression_method.tar')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_with_bad_compression_method_{id}.tar')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -1220,6 +1237,10 @@ def test_system_users_required_privileges():
     assert instance.query("SHOW CREATE ROLE r1") == "CREATE ROLE r1\n"
     assert instance.query("SHOW GRANTS FOR r1") == ""
 
+    instance.query("DROP USER u1")
+    instance.query("DROP ROLE r1")
+    instance.query("DROP USER u2")
+
 
 def test_system_users_async():
     instance.query("CREATE USER u1 IDENTIFIED BY 'qwe123' SETTINGS custom_c = 3")
@@ -1412,6 +1433,8 @@ def test_system_functions():
     assert instance.query("SELECT number, parity_str(number) FROM numbers(3)") == TSV(
         [[0, "even"], [1, "odd"], [2, "even"]]
     )
+    instance.query("DROP FUNCTION linear_equation")
+    instance.query("DROP FUNCTION parity_str")
 
 
 def test_backup_partition():