mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 07:31:57 +00:00
Merge pull request #66986 from qoega/integration-flaky-check-repeat
Add hardening to integration tests flaky check: repeat test case multiple times with same environment.
This commit is contained in:
Ilya Yatsishin
GitHub
commit
db5b4c0e96
@ -29,7 +29,8 @@ CLICKHOUSE_BINARY_PATH = "usr/bin/clickhouse"
|
||||
CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH = "usr/bin/clickhouse-odbc-bridge"
|
||||
CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH = "usr/bin/clickhouse-library-bridge"
|
||||
|
||||
FLAKY_TRIES_COUNT = 10
|
||||
FLAKY_TRIES_COUNT = 10 # run whole pytest several times
|
||||
FLAKY_REPEAT_COUNT = 5 # runs test case in single module several times
|
||||
MAX_TIME_SECONDS = 3600
|
||||
|
||||
MAX_TIME_IN_SANDBOX = 20 * 60 # 20 minutes
|
||||
@ -568,6 +569,7 @@ class ClickhouseIntegrationTestsRunner:
|
||||
tests_in_group,
|
||||
num_tries,
|
||||
num_workers,
|
||||
repeat_count,
|
||||
):
|
||||
try:
|
||||
return self.run_test_group(
|
||||
@ -576,6 +578,7 @@ class ClickhouseIntegrationTestsRunner:
|
||||
tests_in_group,
|
||||
num_tries,
|
||||
num_workers,
|
||||
repeat_count,
|
||||
)
|
||||
except Exception as e:
|
||||
logging.info("Failed to run %s:\n%s", test_group, e)
|
||||
@ -598,6 +601,7 @@ class ClickhouseIntegrationTestsRunner:
|
||||
tests_in_group,
|
||||
num_tries,
|
||||
num_workers,
|
||||
repeat_count,
|
||||
):
|
||||
counters = {
|
||||
"ERROR": [],
|
||||
@ -639,6 +643,7 @@ class ClickhouseIntegrationTestsRunner:
|
||||
|
||||
test_cmd = " ".join([shlex.quote(test) for test in sorted(test_names)])
|
||||
parallel_cmd = f" --parallel {num_workers} " if num_workers > 0 else ""
|
||||
repeat_cmd = f" --count {repeat_count} " if repeat_count > 0 else ""
|
||||
# -r -- show extra test summary:
|
||||
# -f -- (f)ailed
|
||||
# -E -- (E)rror
|
||||
@ -647,7 +652,7 @@ class ClickhouseIntegrationTestsRunner:
|
||||
cmd = (
|
||||
f"cd {repo_path}/tests/integration && "
|
||||
f"timeout --signal=KILL 1h ./runner {self._get_runner_opts()} "
|
||||
f"{image_cmd} -t {test_cmd} {parallel_cmd} -- -rfEps --run-id={i} "
|
||||
f"{image_cmd} -t {test_cmd} {parallel_cmd} {repeat_cmd} -- -rfEps --run-id={i} "
|
||||
f"--color=no --durations=0 {_get_deselect_option(self.should_skip_tests())} "
|
||||
f"| tee {info_path}"
|
||||
)
|
||||
@ -784,7 +789,12 @@ class ClickhouseIntegrationTestsRunner:
|
||||
final_retry += 1
|
||||
logging.info("Running tests for the %s time", i)
|
||||
counters, tests_times, log_paths = self.try_run_test_group(
|
||||
repo_path, "bugfix" if should_fail else "flaky", tests_to_run, 1, 1
|
||||
repo_path,
|
||||
"bugfix" if should_fail else "flaky",
|
||||
tests_to_run,
|
||||
1,
|
||||
1,
|
||||
FLAKY_REPEAT_COUNT,
|
||||
)
|
||||
logs += log_paths
|
||||
if counters["FAILED"]:
|
||||
@ -919,7 +929,7 @@ class ClickhouseIntegrationTestsRunner:
|
||||
for group, tests in items_to_run:
|
||||
logging.info("Running test group %s containing %s tests", group, len(tests))
|
||||
group_counters, group_test_times, log_paths = self.try_run_test_group(
|
||||
repo_path, group, tests, MAX_RETRY, NUM_WORKERS
|
||||
repo_path, group, tests, MAX_RETRY, NUM_WORKERS, 0
|
||||
)
|
||||
total_tests = 0
|
||||
for counter, value in group_counters.items():
|
||||
|
||||
@ -243,6 +243,10 @@ if __name__ == "__main__":
|
||||
"-n", "--parallel", action="store", dest="parallel", help="Parallelism"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--count", action="store", type=int, dest="count", help="Repeat count"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--no-random",
|
||||
action="store",
|
||||
@ -318,6 +322,10 @@ if __name__ == "__main__":
|
||||
parallel_args += "--dist=loadfile"
|
||||
parallel_args += f" -n {args.parallel}".format()
|
||||
|
||||
repeat_args = (
|
||||
f" --count {args.count}" if args.count is not None and args.count > 0 else ""
|
||||
)
|
||||
|
||||
rand_args = ""
|
||||
# if not args.no_random:
|
||||
# rand_args += f"--random-seed={os.getpid()}"
|
||||
@ -409,7 +417,7 @@ if __name__ == "__main__":
|
||||
f"--volume={args.utils_dir}/grpc-client/pb2:/ClickHouse/utils/grpc-client/pb2 "
|
||||
f"--volume=/run:/run/host:ro {dockerd_internal_volume} {env_tags} {env_cleanup} "
|
||||
f"-e DOCKER_CLIENT_TIMEOUT=300 -e COMPOSE_HTTP_TIMEOUT=600 {use_old_analyzer} -e PYTHONUNBUFFERED=1 "
|
||||
f'-e PYTEST_ADDOPTS="{parallel_args} {pytest_opts} {tests_list} {rand_args} -vvv"'
|
||||
f'-e PYTEST_ADDOPTS="{parallel_args} {repeat_args} {pytest_opts} {tests_list} {rand_args} -vvv"'
|
||||
f" {DIND_INTEGRATION_TESTS_IMAGE_NAME}:{args.docker_image_version}"
|
||||
)
|
||||
|
||||
|
||||
@ -1,9 +1,10 @@
|
||||
import pytest
|
||||
import glob
|
||||
import re
|
||||
import random
|
||||
import os.path
|
||||
import pytest
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
import uuid
|
||||
from collections import namedtuple
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.test_tools import assert_eq_with_retry, TSV
|
||||
@ -538,7 +539,8 @@ def test_backup_not_found_or_already_exists():
|
||||
|
||||
|
||||
def test_file_engine():
|
||||
backup_name = f"File('/backups/file/')"
|
||||
id = uuid.uuid4()
|
||||
backup_name = f"File('/backups/file/{id}/')"
|
||||
create_and_fill_table()
|
||||
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
@ -549,6 +551,7 @@ def test_file_engine():
|
||||
|
||||
instance.query(f"RESTORE TABLE test.table FROM {backup_name}")
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
instance.query("DROP TABLE test.table")
|
||||
|
||||
|
||||
def test_database():
|
||||
@ -565,7 +568,8 @@ def test_database():
|
||||
|
||||
|
||||
def test_zip_archive():
|
||||
backup_name = f"Disk('backups', 'archive.zip')"
|
||||
id = uuid.uuid4()
|
||||
backup_name = f"Disk('backups', 'archive_{id}.zip')"
|
||||
create_and_fill_table()
|
||||
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
@ -578,10 +582,12 @@ def test_zip_archive():
|
||||
|
||||
instance.query(f"RESTORE TABLE test.table FROM {backup_name}")
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
instance.query("DROP TABLE test.table")
|
||||
|
||||
|
||||
def test_zip_archive_with_settings():
|
||||
backup_name = f"Disk('backups', 'archive_with_settings.zip')"
|
||||
id = uuid.uuid4()
|
||||
backup_name = f"Disk('backups', 'archive_with_settings_{id}.zip')"
|
||||
create_and_fill_table()
|
||||
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
@ -596,10 +602,12 @@ def test_zip_archive_with_settings():
|
||||
f"RESTORE TABLE test.table FROM {backup_name} SETTINGS password='qwerty'"
|
||||
)
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
instance.query("DROP TABLE test.table")
|
||||
|
||||
|
||||
def test_zip_archive_with_bad_compression_method():
|
||||
backup_name = f"Disk('backups', 'archive_with_bad_compression_method.zip')"
|
||||
id = uuid.uuid4()
|
||||
backup_name = f"Disk('backups', 'archive_with_bad_compression_method_{id}.zip')"
|
||||
create_and_fill_table()
|
||||
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
@ -617,7 +625,8 @@ def test_zip_archive_with_bad_compression_method():
|
||||
|
||||
|
||||
def test_tar_archive():
|
||||
backup_name = f"Disk('backups', 'archive.tar')"
|
||||
id = uuid.uuid4()
|
||||
backup_name = f"Disk('backups', 'archive_{id}.tar')"
|
||||
create_and_fill_table()
|
||||
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
@ -630,10 +639,12 @@ def test_tar_archive():
|
||||
|
||||
instance.query(f"RESTORE TABLE test.table FROM {backup_name}")
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
instance.query("DROP TABLE test.table")
|
||||
|
||||
|
||||
def test_tar_bz2_archive():
|
||||
backup_name = f"Disk('backups', 'archive.tar.bz2')"
|
||||
id = uuid.uuid4()
|
||||
backup_name = f"Disk('backups', 'archive_{id}.tar.bz2')"
|
||||
create_and_fill_table()
|
||||
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
@ -649,7 +660,8 @@ def test_tar_bz2_archive():
|
||||
|
||||
|
||||
def test_tar_gz_archive():
|
||||
backup_name = f"Disk('backups', 'archive.tar.gz')"
|
||||
id = uuid.uuid4()
|
||||
backup_name = f"Disk('backups', 'archive_{id}.tar.gz')"
|
||||
create_and_fill_table()
|
||||
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
@ -665,7 +677,8 @@ def test_tar_gz_archive():
|
||||
|
||||
|
||||
def test_tar_lzma_archive():
|
||||
backup_name = f"Disk('backups', 'archive.tar.lzma')"
|
||||
id = uuid.uuid4()
|
||||
backup_name = f"Disk('backups', 'archive_{id}.tar.lzma')"
|
||||
create_and_fill_table()
|
||||
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
@ -681,7 +694,8 @@ def test_tar_lzma_archive():
|
||||
|
||||
|
||||
def test_tar_zst_archive():
|
||||
backup_name = f"Disk('backups', 'archive.tar.zst')"
|
||||
id = uuid.uuid4()
|
||||
backup_name = f"Disk('backups', 'archive_{id}.tar.zst')"
|
||||
create_and_fill_table()
|
||||
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
@ -697,7 +711,8 @@ def test_tar_zst_archive():
|
||||
|
||||
|
||||
def test_tar_xz_archive():
|
||||
backup_name = f"Disk('backups', 'archive.tar.xz')"
|
||||
id = uuid.uuid4()
|
||||
backup_name = f"Disk('backups', 'archive_{id}.tar.xz')"
|
||||
create_and_fill_table()
|
||||
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
@ -713,7 +728,8 @@ def test_tar_xz_archive():
|
||||
|
||||
|
||||
def test_tar_archive_with_password():
|
||||
backup_name = f"Disk('backups', 'archive_with_password.tar')"
|
||||
id = uuid.uuid4()
|
||||
backup_name = f"Disk('backups', 'archive_with_password_{id}.tar')"
|
||||
create_and_fill_table()
|
||||
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
@ -731,7 +747,8 @@ def test_tar_archive_with_password():
|
||||
|
||||
|
||||
def test_tar_archive_with_bad_compression_method():
|
||||
backup_name = f"Disk('backups', 'archive_with_bad_compression_method.tar')"
|
||||
id = uuid.uuid4()
|
||||
backup_name = f"Disk('backups', 'archive_with_bad_compression_method_{id}.tar')"
|
||||
create_and_fill_table()
|
||||
|
||||
assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
|
||||
@ -1220,6 +1237,10 @@ def test_system_users_required_privileges():
|
||||
assert instance.query("SHOW CREATE ROLE r1") == "CREATE ROLE r1\n"
|
||||
assert instance.query("SHOW GRANTS FOR r1") == ""
|
||||
|
||||
instance.query("DROP USER u1")
|
||||
instance.query("DROP ROLE r1")
|
||||
instance.query("DROP USER u2")
|
||||
|
||||
|
||||
def test_system_users_async():
|
||||
instance.query("CREATE USER u1 IDENTIFIED BY 'qwe123' SETTINGS custom_c = 3")
|
||||
@ -1412,6 +1433,8 @@ def test_system_functions():
|
||||
assert instance.query("SELECT number, parity_str(number) FROM numbers(3)") == TSV(
|
||||
[[0, "even"], [1, "odd"], [2, "even"]]
|
||||
)
|
||||
instance.query("DROP FUNCTION linear_equation")
|
||||
instance.query("DROP FUNCTION parity_str")
|
||||
|
||||
|
||||
def test_backup_partition():
|
||||
|
||||
Loading…
Reference in New Issue
Block a user