Backport #69013 to 24.8: Fix test_role & test_keeper_s3_snapshot integration tests

tests/integration/test_keeper_s3_snapshot/configs/keeper_config1.xml

@@ -5,6 +5,7 @@
             <access_key_id>minio</access_key_id>
             <secret_access_key>minio123</secret_access_key>
         </s3_snapshot>
+        <use_cluster>false</use_cluster>
         <tcp_port>9181</tcp_port>
         <server_id>1</server_id>
         <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>

tests/integration/test_keeper_s3_snapshot/configs/keeper_config2.xml

@@ -5,6 +5,7 @@
             <access_key_id>minio</access_key_id>
             <secret_access_key>minio123</secret_access_key>
         </s3_snapshot>
+        <use_cluster>false</use_cluster>
         <tcp_port>9181</tcp_port>
         <server_id>2</server_id>
         <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>

tests/integration/test_keeper_s3_snapshot/configs/keeper_config3.xml

@@ -5,6 +5,7 @@
             <access_key_id>minio</access_key_id>
             <secret_access_key>minio123</secret_access_key>
         </s3_snapshot>
+        <use_cluster>false</use_cluster>
         <tcp_port>9181</tcp_port>
         <server_id>3</server_id>
         <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>

tests/integration/test_keeper_s3_snapshot/test.py

@@ -2,6 +2,9 @@ import pytest
 from helpers.cluster import ClickHouseCluster
 from time import sleep
 from retry import retry
+from multiprocessing.dummy import Pool
+import helpers.keeper_utils as keeper_utils
+from minio.deleteobjects import DeleteObject
 
 from kazoo.client import KazooClient
 
@@ -75,7 +78,18 @@ def wait_node(node):
         raise Exception("Can't wait node", node.name, "to become ready")
 
 
+def delete_keeper_snapshots_logs(nodex):
+    nodex.exec_in_container(
+        [
+            "bash",
+            "-c",
+            "rm -rf /var/lib/clickhouse/coordination/log /var/lib/clickhouse/coordination/snapshots",
+        ]
+    )
+
+
 def test_s3_upload(started_cluster):
+
     node1_zk = get_fake_zk(node1.name)
 
     # we defined in configs snapshot_distance as 50
@@ -89,6 +103,11 @@ def test_s3_upload(started_cluster):
             for obj in list(cluster.minio_client.list_objects("snapshots"))
         ]
 
+    def delete_s3_snapshots():
+        snapshots = cluster.minio_client.list_objects("snapshots")
+        for s in snapshots:
+            cluster.minio_client.remove_object("snapshots", s.object_name)
+
     # Keeper sends snapshots asynchornously, hence we need to retry.
     @retry(AssertionError, tries=10, delay=2)
     def _check_snapshots():
@@ -125,3 +144,26 @@ def test_s3_upload(started_cluster):
     )
 
     destroy_zk_client(node2_zk)
+    node2.stop_clickhouse()
+    delete_keeper_snapshots_logs(node2)
+    node3.stop_clickhouse()
+    delete_keeper_snapshots_logs(node3)
+    delete_keeper_snapshots_logs(node1)
+    p = Pool(3)
+    waiters = []
+
+    def start_clickhouse(node):
+        node.start_clickhouse()
+
+    waiters.append(p.apply_async(start_clickhouse, args=(node1,)))
+    waiters.append(p.apply_async(start_clickhouse, args=(node2,)))
+    waiters.append(p.apply_async(start_clickhouse, args=(node3,)))
+
+    delete_s3_snapshots()  # for next iteration
+
+    for waiter in waiters:
+        waiter.wait()
+
+    keeper_utils.wait_until_connected(cluster, node1)
+    keeper_utils.wait_until_connected(cluster, node2)
+    keeper_utils.wait_until_connected(cluster, node3)

tests/integration/test_role/test.py

@@ -629,5 +629,6 @@ def test_roles_cache():
     check()
 
     instance.query("DROP USER " + ", ".join(users))
+    if roles:
         instance.query("DROP ROLE " + ", ".join(roles))
     instance.query("DROP TABLE tbl")