From c7781e24d51ba9f32bc7543c50bcffd23153a349 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Jun 2021 11:02:27 +0300 Subject: [PATCH] Fix some flaky tests --- tests/integration/helpers/cluster.py | 12 +++++++++--- tests/integration/pytest.ini | 2 +- tests/integration/test_merge_tree_s3/test.py | 4 ++-- .../test_reload_max_table_size_to_drop/test.py | 5 +++-- tests/integration/test_storage_s3/test.py | 5 +++-- tests/integration/test_system_flush_logs/test.py | 5 ++--- 6 files changed, 20 insertions(+), 13 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 7888e07587c..eb7ed55c7a2 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -29,7 +29,6 @@ from dict2xml import dict2xml from kazoo.client import KazooClient from kazoo.exceptions import KazooException from minio import Minio -from minio.deleteobjects import DeleteObject from helpers.test_tools import assert_eq_with_retry import docker @@ -172,6 +171,13 @@ def enable_consistent_hash_plugin(rabbitmq_id): p.communicate() return p.returncode == 0 +def get_instances_dir(): + if 'INTEGRATION_TESTS_RUN_ID' in os.environ: + return '_instances_' + shlex.quote(os.environ['INTEGRATION_TESTS_RUN_ID']) + else: + return '_instances' + + class ClickHouseCluster: """ClickHouse cluster with several instances and (possibly) ZooKeeper. @@ -1232,8 +1238,8 @@ class ClickHouseCluster: for bucket in buckets: if minio_client.bucket_exists(bucket): delete_object_list = map( - lambda x: DeleteObject(x.object_name), - minio_client.list_objects(bucket, recursive=True), + lambda x: x.object_name, + minio_client.list_objects_v2(bucket, recursive=True), ) errors = minio_client.remove_objects(bucket, delete_object_list) for error in errors: diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini index 737a37ee9d0..6d451adf7eb 100644 --- a/tests/integration/pytest.ini +++ b/tests/integration/pytest.ini @@ -1,6 +1,6 @@ [pytest] python_files = test*.py -norecursedirs = _instances +norecursedirs = _instances* timeout = 1800 junit_duration_report = call junit_suite_name = integration diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index 4b30287ca1e..31df6dff374 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -6,7 +6,7 @@ import threading import os import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, get_instances_dir # By default the exceptions that was throwed in threads will be ignored @@ -30,7 +30,7 @@ class SafeThread(threading.Thread): SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -CONFIG_PATH = os.path.join(SCRIPT_DIR, './_instances/node/configs/config.d/storage_conf.xml') +CONFIG_PATH = os.path.join(SCRIPT_DIR, './{}/node/configs/config.d/storage_conf.xml'.format(get_instances_dir())) def replace_config(old, new): diff --git a/tests/integration/test_reload_max_table_size_to_drop/test.py b/tests/integration/test_reload_max_table_size_to_drop/test.py index 5f2083d742e..7e7219088b8 100644 --- a/tests/integration/test_reload_max_table_size_to_drop/test.py +++ b/tests/integration/test_reload_max_table_size_to_drop/test.py @@ -2,13 +2,14 @@ import os import time import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, get_instances_dir cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', main_configs=["configs/max_table_size_to_drop.xml"]) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -CONFIG_PATH = os.path.join(SCRIPT_DIR, './_instances/node/configs/config.d/max_table_size_to_drop.xml') + +CONFIG_PATH = os.path.join(SCRIPT_DIR, './{}/node/configs/config.d/max_table_size_to_drop.xml'.format(get_instances_dir())) @pytest.fixture(scope="module") diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 545ca4256f3..52b021a07c5 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -9,12 +9,13 @@ import time import helpers.client import pytest -from helpers.cluster import ClickHouseCluster, ClickHouseInstance +from helpers.cluster import ClickHouseCluster, ClickHouseInstance, get_instances_dir MINIO_INTERNAL_PORT = 9001 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -CONFIG_PATH = os.path.join(SCRIPT_DIR, './_instances/dummy/configs/config.d/defaultS3.xml') + +CONFIG_PATH = os.path.join(SCRIPT_DIR, './{}/dummy/configs/config.d/defaultS3.xml'.format(get_instances_dir())) # Creates S3 bucket for tests and allows anonymous read-write access to it. diff --git a/tests/integration/test_system_flush_logs/test.py b/tests/integration/test_system_flush_logs/test.py index 6dc843e101a..a1a21f24099 100644 --- a/tests/integration/test_system_flush_logs/test.py +++ b/tests/integration/test_system_flush_logs/test.py @@ -20,9 +20,8 @@ system_logs = [ ('system.metric_log', 1), ] -# Default timeout for flush is 60 -# decrease timeout for the test to show possible issues. -timeout = pytest.mark.timeout(30) +# 60 -- Default timeout for flush +timeout = pytest.mark.timeout(60) @pytest.fixture(scope='module', autouse=True)