From c03510cf6ba3442ebfff26da3665d5b40b0adeb4 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 6 Jan 2023 12:21:35 +0100 Subject: [PATCH 1/2] tests/integration: add query_and_get_error_with_retry() helper Signed-off-by: Azat Khuzhin --- tests/integration/helpers/cluster.py | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 5a9f8b20254..7b53e033c9d 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -3225,6 +3225,40 @@ class ClickHouseInstance: database=database, ) + def query_and_get_error_with_retry( + self, + sql, + stdin=None, + timeout=None, + settings=None, + user=None, + password=None, + database=None, + retry_count=20, + sleep_time=0.5, + ): + logging.debug(f"Executing query {sql} on {self.name}") + result = None + for i in range(retry_count): + try: + result = self.client.query_and_get_error( + sql, + stdin=stdin, + timeout=timeout, + settings=settings, + user=user, + password=password, + database=database, + ) + time.sleep(sleep_time) + except QueryRuntimeException as ex: + logging.debug("Retry {} got exception {}".format(i + 1, ex)) + time.sleep(sleep_time) + + if result is not None: + return result + raise Exception("Query {sql} did not fail".format(sql)) + # The same as query_and_get_error but ignores successful query. def query_and_get_answer_with_error( self, From 7859346e40bfa08ebb9b8719e88552dc91b885d3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 6 Jan 2023 12:23:11 +0100 Subject: [PATCH 2/2] tests: fix test_replicated_users flakiness Alter for users/quotas/... does not wait until all replicas will be up to date, so use *_with_retries() helperx. CI: https://s3.amazonaws.com/clickhouse-test-reports/44922/bd885be9229cf47752c5c98392f09129261550f9/integration_tests__tsan__[6/6].html Signed-off-by: Azat Khuzhin --- tests/integration/test_replicated_users/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_replicated_users/test.py b/tests/integration/test_replicated_users/test.py index 4fb2408c46f..1c73fc19c01 100644 --- a/tests/integration/test_replicated_users/test.py +++ b/tests/integration/test_replicated_users/test.py @@ -58,7 +58,7 @@ def test_create_replicated(started_cluster, entity): node1.query(f"CREATE {entity.keyword} {entity.name} {entity.options}") assert ( f"cannot insert because {entity.keyword.lower()} `{entity.name}{entity.options}` already exists in replicated" - in node2.query_and_get_error( + in node2.query_and_get_error_with_retry( f"CREATE {entity.keyword} {entity.name} {entity.options}" ) ) @@ -68,7 +68,7 @@ def test_create_replicated(started_cluster, entity): @pytest.mark.parametrize("entity", entities, ids=get_entity_id) def test_create_and_delete_replicated(started_cluster, entity): node1.query(f"CREATE {entity.keyword} {entity.name} {entity.options}") - node2.query(f"DROP {entity.keyword} {entity.name} {entity.options}") + node2.query_with_retry(f"DROP {entity.keyword} {entity.name} {entity.options}") @pytest.mark.parametrize("entity", entities, ids=get_entity_id) @@ -93,7 +93,7 @@ def test_create_replicated_if_not_exists_on_cluster(started_cluster, entity): @pytest.mark.parametrize("entity", entities, ids=get_entity_id) def test_rename_replicated(started_cluster, entity): node1.query(f"CREATE {entity.keyword} {entity.name} {entity.options}") - node2.query( + node2.query_with_retry( f"ALTER {entity.keyword} {entity.name} {entity.options} RENAME TO {entity.name}2" ) node1.query(f"DROP {entity.keyword} {entity.name}2 {entity.options}")