Revert "Backport #70146 to 24.8: Upgrade integration-runner image"

docker/reqgenerator.py

@@ -7,24 +7,14 @@ import os
 import sys
 
 
-def build_docker_deps(image_name: str, imagedir: str) -> None:
+def build_docker_deps(image_name, imagedir):
-    print("Fetch the newest manifest for", image_name)
+    cmd = f"""docker run  --entrypoint "/bin/bash" {image_name} -c "pip install pipdeptree 2>/dev/null 1>/dev/null && pipdeptree --freeze  --warn silence | sed 's/ \+//g' | sort | uniq" > {imagedir}/requirements.txt"""
-    pip_cmd = (
-        "pip install pipdeptree 2>/dev/null 1>/dev/null && pipdeptree --freeze "
-        "--warn silence --exclude pipdeptree"
-    )
-    # /=/!d - remove dependencies without pin
-    # ubuntu - ignore system packages
-    # \s - remove spaces
-    sed = r"sed '/==/!d; /==.*+ubuntu/d; s/\s//g'"
-    cmd = rf"""docker run --rm --entrypoint "/bin/bash" {image_name} -c "{pip_cmd} | {sed} | sort -u" > {imagedir}/requirements.txt"""
-    print("Running the command:", cmd)
     subprocess.check_call(cmd, shell=True)
 
 
 def check_docker_file_install_with_pip(filepath):
     image_name = None
-    with open(filepath, "r", encoding="utf-8") as f:
+    with open(filepath, "r") as f:
         for line in f:
             if "docker build" in line:
                 arr = line.split(" ")
@@ -35,7 +25,7 @@ def check_docker_file_install_with_pip(filepath):
     return image_name, False
 
 
-def process_affected_images(images_dir: str) -> None:
+def process_affected_images(images_dir):
     for root, _dirs, files in os.walk(images_dir):
         for f in files:
             if f == "Dockerfile":

docker/test/integration/runner/Dockerfile

@@ -48,7 +48,7 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
     && add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" \
     && apt-get update \
     && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
-        docker-ce='5:23.*' docker-compose-plugin='2.29.*' \
+        docker-ce='5:23.*' \
     && rm -rf \
         /var/lib/apt/lists/* \
         /var/cache/debconf \

docker/test/integration/runner/requirements.txt

@@ -1,13 +1,15 @@
 PyHDFS==0.3.1
-PyJWT==2.4.0
+PyJWT==2.3.0
-PyMySQL==1.1.1
+PyMySQL==1.1.0
 PyNaCl==1.5.0
+PyYAML==5.3.1
 SecretStorage==3.3.1
 argon2-cffi-bindings==21.2.0
 argon2-cffi==23.1.0
 async-timeout==4.0.3
 asyncio==3.4.3
-avro==1.11.3
+attrs==23.2.0
+avro==1.10.2
 azure-core==1.30.1
 azure-storage-blob==12.19.0
 bcrypt==4.1.3
@@ -22,13 +24,18 @@ cffi==1.16.0
 charset-normalizer==3.3.2
 click==8.1.7
 confluent-kafka==2.3.0
-cryptography==42.0.0
+cryptography==3.4.8
 dbus-python==1.2.18
+decorator==5.1.1
 delta-spark==2.3.0
-deltalake==0.16.0
 dict2xml==1.7.4
 dicttoxml==1.7.16
+distro-info==1.1+ubuntu0.2
+distro==1.7.0
+docker-compose==1.29.2
 docker==6.1.3
+dockerpty==0.4.1
+docopt==0.6.2
 exceptiongroup==1.2.1
 execnet==2.1.1
 geomet==0.2.1.post1
@@ -42,6 +49,7 @@ iniconfig==2.0.0
 isodate==0.6.1
 jeepney==0.7.1
 jmespath==1.0.1
+jsonschema==3.2.0
 jwcrypto==1.5.6
 kafka-python==2.0.2
 kazoo==2.9.0
@@ -55,22 +63,23 @@ lz4==4.3.3
 minio==7.2.3
 more-itertools==8.10.0
 nats-py==2.6.0
-numpy==2.1.0
 oauthlib==3.2.0
 packaging==24.0
 paramiko==3.4.0
 pika==1.2.0
 pip==24.1.1
+pipdeptree==2.23.0
 pluggy==1.5.0
 protobuf==4.25.2
 psycopg2-binary==2.9.6
 py4j==0.10.9.5
-pyarrow-hotfix==0.6
+py==1.11.0
 pyarrow==17.0.0
 pycparser==2.22
 pycryptodome==3.20.0
 pymongo==3.11.0
 pyparsing==2.4.7
+pyrsistent==0.20.0
 pyspark==3.3.2
 pyspnego==0.10.2
 pytest-order==1.0.0
@@ -80,22 +89,28 @@ pytest-reportlog==0.4.0
 pytest-timeout==2.2.0
 pytest-xdist==3.5.0
 pytest==7.4.4
+python-apt==2.4.0+ubuntu3
 python-dateutil==2.9.0.post0
+python-dotenv==0.21.1
 pytz==2023.3.post1
 redis==5.0.1
 requests-kerberos==0.14.0
 requests==2.31.0
+retry==0.9.2
 s3transfer==0.10.1
-setuptools==70.0.0
+setuptools==59.6.0
 simplejson==3.19.2
 six==1.16.0
 soupsieve==2.5
+texttable==1.7.0
 tomli==2.0.1
 typing_extensions==4.11.0
 tzlocal==2.1
 unattended-upgrades==0.1
 urllib3==2.0.7
 wadllib==1.3.6
-websocket-client==1.8.0
+websocket-client==0.59.0
-wheel==0.38.1
+wheel==0.37.1
 zipp==1.0.0
+deltalake==0.16.0
+

tests/integration/README.md

@@ -14,7 +14,7 @@ Don't use Docker from your system repository.
 
 * [pip](https://pypi.python.org/pypi/pip) and `libpq-dev`. To install: `sudo apt-get install python3-pip libpq-dev zlib1g-dev libcrypto++-dev libssl-dev libkrb5-dev python3-dev`
 * [py.test](https://docs.pytest.org/) testing framework. To install: `sudo -H pip install pytest`
-* [docker compose](https://docs.docker.com/compose/) and additional python libraries. To install:
+* [docker-compose](https://docs.docker.com/compose/) and additional python libraries. To install:
 
 ```bash
 sudo -H pip install \
@@ -24,6 +24,7 @@ sudo -H pip install \
     confluent-kafka \
     dicttoxml \
     docker \
+    docker-compose \
     grpcio \
     grpcio-tools \
     kafka-python \
@@ -47,7 +48,7 @@ sudo -H pip install \
     nats-py
 ```
 
-(highly not recommended) If you really want to use OS packages on modern debian/ubuntu instead of "pip": `sudo apt install -y docker docker-compose-v2 python3-pytest python3-dicttoxml python3-docker python3-pymysql python3-protobuf python3-pymongo python3-tzlocal python3-kazoo python3-psycopg2 kafka-python python3-pytest-timeout python3-minio`
+(highly not recommended) If you really want to use OS packages on modern debian/ubuntu instead of "pip": `sudo apt install -y docker docker-compose python3-pytest python3-dicttoxml python3-docker python3-pymysql python3-protobuf python3-pymongo python3-tzlocal python3-kazoo python3-psycopg2 kafka-python python3-pytest-timeout python3-minio`
 
 Some tests have other dependencies, e.g. spark. See docker/test/integration/runner/Dockerfile for how to install those. See docker/test/integration/runner/dockerd-entrypoint.sh for environment variables that need to be set (e.g. JAVA_PATH).
 

tests/integration/compose/docker_compose_azurite.yml

@@ -1,3 +1,5 @@
+version: '2.3'
+
 services:
   azurite1:
     image: mcr.microsoft.com/azure-storage/azurite

tests/integration/compose/docker_compose_cassandra.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     cassandra1:
         image: cassandra:4.0

tests/integration/compose/docker_compose_clickhouse.yml

@@ -1,4 +1,5 @@
-# Used to pre-pull images with docker compose
+version: '2.3'
+# Used to pre-pull images with docker-compose
 services:
   clickhouse1:
     image: clickhouse/integration-test

tests/integration/compose/docker_compose_coredns.yml

@@ -1,3 +1,5 @@
+version: "2.3"
+
 services:
   coredns:
     image: coredns/coredns:1.9.3 # :latest broke this test

tests/integration/compose/docker_compose_dotnet_client.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
   dotnet1:
     image: clickhouse/dotnet-client:${DOCKER_DOTNET_CLIENT_TAG:-latest}

tests/integration/compose/docker_compose_hdfs.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     hdfs1:
         image: prasanthj/docker-hadoop:2.6.0

tests/integration/compose/docker_compose_hive.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     hdfs1:
         image: lgboustc/hive_test:v2.0

tests/integration/compose/docker_compose_jdbc_bridge.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
   bridge1:
     image: clickhouse/jdbc-bridge

tests/integration/compose/docker_compose_kafka.yml

@@ -1,3 +1,5 @@
+version: '2.3'
+
 services:
   kafka_zookeeper:
     image: zookeeper:3.4.9

tests/integration/compose/docker_compose_keeper.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     zoo1:
         image: ${image:-clickhouse/integration-test}

tests/integration/compose/docker_compose_kerberized_hdfs.yml

@@ -1,3 +1,5 @@
+version: '2.3'
+
 services:
   kerberizedhdfs1:
     cap_add:

tests/integration/compose/docker_compose_kerberized_kafka.yml

@@ -1,3 +1,5 @@
+version: '2.3'
+
 services:
   kafka_kerberized_zookeeper:
     image: confluentinc/cp-zookeeper:5.2.0

tests/integration/compose/docker_compose_kerberos_kdc.yml

@@ -1,3 +1,5 @@
+version: '2.3'
+
 services:
   kerberoskdc:
     image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest}

tests/integration/compose/docker_compose_ldap.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     openldap:
         image: bitnami/openldap:2.6.6

tests/integration/compose/docker_compose_minio.yml

@@ -1,3 +1,5 @@
+version: '2.3'
+
 services:
   minio1:
     image: minio/minio:RELEASE.2024-07-31T05-46-26Z

tests/integration/compose/docker_compose_mongo.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     mongo1:
         image: mongo:5.0

tests/integration/compose/docker_compose_mysql.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     mysql57:
         image: mysql:5.7

tests/integration/compose/docker_compose_mysql_8_0.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     mysql80:
         image: mysql:8.0

tests/integration/compose/docker_compose_mysql_client.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
   mysql_client:
     image: mysql:8.0

tests/integration/compose/docker_compose_mysql_cluster.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     mysql2:
         image: mysql:8.0

tests/integration/compose/docker_compose_mysql_golang_client.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
   golang1:
     image: clickhouse/mysql-golang-client:${DOCKER_MYSQL_GOLANG_CLIENT_TAG:-latest}

tests/integration/compose/docker_compose_mysql_java_client.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
   java1:
     image: clickhouse/mysql-java-client:${DOCKER_MYSQL_JAVA_CLIENT_TAG:-latest}

tests/integration/compose/docker_compose_mysql_js_client.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
   mysqljs1:
     image: clickhouse/mysql-js-client:${DOCKER_MYSQL_JS_CLIENT_TAG:-latest}

tests/integration/compose/docker_compose_mysql_php_client.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
   php1:
     image: clickhouse/mysql-php-client:${DOCKER_MYSQL_PHP_CLIENT_TAG:-latest}

tests/integration/compose/docker_compose_nats.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
   nats1:
     image: nats

tests/integration/compose/docker_compose_net.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 networks:
   default:
     driver: bridge

tests/integration/compose/docker_compose_nginx.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     # nginx server to host static files.
     # Accepts only PUT data by test.com/path and GET already existing data on test.com/path.

tests/integration/compose/docker_compose_postgres.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     postgres1:
         image: postgres

tests/integration/compose/docker_compose_postgres_cluster.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     postgres2:
         image: postgres

tests/integration/compose/docker_compose_postgresql.yml

@@ -1,3 +1,4 @@
+version: '2.2'
 services:
   psql:
     image: postgres:12.2-alpine

tests/integration/compose/docker_compose_postgresql_java_client.yml

@@ -1,3 +1,4 @@
+version: '2.2'
 services:
   java:
     image: clickhouse/postgresql-java-client:${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:-latest}

tests/integration/compose/docker_compose_prometheus.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
   prometheus_writer:
     image: prom/prometheus:v2.50.1

tests/integration/compose/docker_compose_rabbitmq.yml

@@ -1,3 +1,5 @@
+version: '2.3'
+
 services:
     rabbitmq1:
         image: rabbitmq:3.12.6-alpine

tests/integration/compose/docker_compose_redis.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     redis1:
         image: redis

tests/integration/compose/docker_compose_zookeeper.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     zoo1:
         image: zookeeper:3.6.2

tests/integration/compose/docker_compose_zookeeper_secure.yml

@@ -1,3 +1,4 @@
+version: '2.3'
 services:
     zoo1:
         image: zookeeper:3.6.2

tests/integration/conftest.py

@@ -89,7 +89,7 @@ def cleanup_environment():
                     nothrow=True,
                 )
                 logging.debug("Unstopped containers killed")
-                r = run_and_check(["docker", "compose", "ps", "--services", "--all"])
+                r = run_and_check(["docker-compose", "ps", "--services", "--all"])
                 logging.debug("Docker ps before start:%s", r.stdout)
         else:
             logging.debug("No running containers")

tests/integration/helpers/cluster.py

@@ -1,63 +1,61 @@
 import base64
 import errno
+from functools import cache
 import http.client
 import logging
 import os
 import platform
+import stat
+import os.path as p
 import pprint
 import pwd
 import re
-import shlex
 import shutil
 import socket
-import stat
 import subprocess
 import time
 import traceback
 import urllib.parse
-from functools import cache
+import shlex
-from os import path as p
-from pathlib import Path
-from typing import List, Sequence, Tuple, Union
-
-import requests
 import urllib3
+import requests
 
 try:
     # Please, add modules that required for specific tests only here.
     # So contributors will be able to run most tests locally
     # without installing tons of unneeded packages that may be not so easy to install.
     import asyncio
-    import ssl
+    from cassandra.policies import RoundRobinPolicy
-
     import cassandra.cluster
-    import nats
     import psycopg2
+    from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
     import pymongo
     import pymysql
-    from cassandra.policies import RoundRobinPolicy
+    import nats
+    import ssl
     from confluent_kafka.avro.cached_schema_registry_client import (
         CachedSchemaRegistryClient,
     )
-    from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
-
     from .hdfs_api import HDFSApi  # imports requests_kerberos
 except Exception as e:
     logging.warning(f"Cannot import some modules, some tests may not work: {e}")
 
-import docker
 from dict2xml import dict2xml
-from helpers import pytest_xdist_logging_to_separate_files
-from helpers.client import QueryRuntimeException
-from helpers.test_tools import assert_eq_with_retry, exec_query_with_retry
 from kazoo.client import KazooClient
 from kazoo.exceptions import KazooException
 from minio import Minio
 
+from helpers.test_tools import assert_eq_with_retry, exec_query_with_retry
+from helpers import pytest_xdist_logging_to_separate_files
+from helpers.client import QueryRuntimeException
+
+import docker
+
 from .client import Client
-from .config_cluster import *
 from .retry_decorator import retry
 
+from .config_cluster import *
+
 HELPERS_DIR = p.dirname(__file__)
 CLICKHOUSE_ROOT_DIR = p.join(p.dirname(__file__), "../../..")
 LOCAL_DOCKER_COMPOSE_DIR = p.join(CLICKHOUSE_ROOT_DIR, "tests/integration/compose/")
@@ -81,7 +79,7 @@ CLICKHOUSE_CI_MIN_TESTED_VERSION = "23.3"
 
 # to create docker-compose env file
 def _create_env_file(path, variables):
-    logging.debug("Env %s stored in %s", variables, path)
+    logging.debug(f"Env {variables} stored in {path}")
     with open(path, "w") as f:
         for var, value in list(variables.items()):
             f.write("=".join([var, value]) + "\n")
@@ -89,7 +87,7 @@ def _create_env_file(path, variables):
 
 
 def run_and_check(
-    args: Union[Sequence[str], str],
+    args,
     env=None,
     shell=False,
     stdout=subprocess.PIPE,
@@ -97,16 +95,7 @@ def run_and_check(
     timeout=300,
     nothrow=False,
     detach=False,
-) -> str:
+):
-    if shell:
-        if isinstance(args, str):
-            shell_args = args
-        else:
-            shell_args = next(a for a in args)
-    else:
-        shell_args = " ".join(args)
-
-    logging.debug("Command:[%s]", shell_args)
     if detach:
         subprocess.Popen(
             args,
@@ -115,31 +104,26 @@ def run_and_check(
             env=env,
             shell=shell,
         )
-        return ""
+        return
 
+    logging.debug(f"Command:{args}")
     res = subprocess.run(
-        args,
+        args, stdout=stdout, stderr=stderr, env=env, shell=shell, timeout=timeout
-        stdout=stdout,
-        stderr=stderr,
-        env=env,
-        shell=shell,
-        timeout=timeout,
-        check=False,
     )
     out = res.stdout.decode("utf-8", "ignore")
     err = res.stderr.decode("utf-8", "ignore")
     # check_call(...) from subprocess does not print stderr, so we do it manually
     for outline in out.splitlines():
-        logging.debug("Stdout:%s", outline)
+        logging.debug(f"Stdout:{outline}")
     for errline in err.splitlines():
-        logging.debug("Stderr:%s", errline)
+        logging.debug(f"Stderr:{errline}")
     if res.returncode != 0:
-        logging.debug("Exitcode:%s", res.returncode)
+        logging.debug(f"Exitcode:{res.returncode}")
         if env:
-            logging.debug("Env:%s", env)
+            logging.debug(f"Env:{env}")
         if not nothrow:
             raise Exception(
-                f"Command [{shell_args}] return non-zero code {res.returncode}: {res.stderr.decode('utf-8')}"
+                f"Command {args} return non-zero code {res.returncode}: {res.stderr.decode('utf-8')}"
             )
     return out
 
@@ -197,11 +181,6 @@ class PortPoolManager:
         self.used_ports.clear()
 
 
-def docker_exec(*args: str) -> Tuple[str, ...]:
-    "Function to ease the `docker exec -i...`"
-    return ("docker", "exec", "-i", *args)
-
-
 def retry_exception(num, delay, func, exception=Exception, *args, **kwargs):
     """
     Retry if `func()` throws, `num` times.
@@ -259,7 +238,10 @@ def get_docker_compose_path():
 
 def check_kafka_is_available(kafka_id, kafka_port):
     p = subprocess.Popen(
-        docker_exec(
+        (
+            "docker",
+            "exec",
+            "-i",
             kafka_id,
             "/usr/bin/kafka-broker-api-versions",
             "--bootstrap-server",
@@ -274,7 +256,14 @@ def check_kafka_is_available(kafka_id, kafka_port):
 
 def check_kerberos_kdc_is_available(kerberos_kdc_id):
     p = subprocess.Popen(
-        docker_exec(kerberos_kdc_id, "/etc/rc.d/init.d/krb5kdc", "status"),
+        (
+            "docker",
+            "exec",
+            "-i",
+            kerberos_kdc_id,
+            "/etc/rc.d/init.d/krb5kdc",
+            "status",
+        ),
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
     )
@@ -284,7 +273,7 @@ def check_kerberos_kdc_is_available(kerberos_kdc_id):
 
 def check_postgresql_java_client_is_available(postgresql_java_client_id):
     p = subprocess.Popen(
-        docker_exec(postgresql_java_client_id, "java", "-version"),
+        ("docker", "exec", "-i", postgresql_java_client_id, "java", "-version"),
         stdout=subprocess.PIPE,
     )
     p.communicate()
@@ -293,9 +282,12 @@ def check_postgresql_java_client_is_available(postgresql_java_client_id):
 
 def check_rabbitmq_is_available(rabbitmq_id, cookie):
     p = subprocess.Popen(
-        docker_exec(
+        (
+            "docker",
+            "exec",
             "-e",
             f"RABBITMQ_ERLANG_COOKIE={cookie}",
+            "-i",
             rabbitmq_id,
             "rabbitmqctl",
             "await_startup",
@@ -308,9 +300,12 @@ def check_rabbitmq_is_available(rabbitmq_id, cookie):
 
 def rabbitmq_debuginfo(rabbitmq_id, cookie):
     p = subprocess.Popen(
-        docker_exec(
+        (
+            "docker",
+            "exec",
             "-e",
             f"RABBITMQ_ERLANG_COOKIE={cookie}",
+            "-i",
             rabbitmq_id,
             "rabbitmq-diagnostics",
             "status",
@@ -320,9 +315,12 @@ def rabbitmq_debuginfo(rabbitmq_id, cookie):
     p.communicate()
 
     p = subprocess.Popen(
-        docker_exec(
+        (
+            "docker",
+            "exec",
             "-e",
             f"RABBITMQ_ERLANG_COOKIE={cookie}",
+            "-i",
             rabbitmq_id,
             "rabbitmq-diagnostics",
             "listeners",
@@ -332,9 +330,12 @@ def rabbitmq_debuginfo(rabbitmq_id, cookie):
     p.communicate()
 
     p = subprocess.Popen(
-        docker_exec(
+        (
+            "docker",
+            "exec",
             "-e",
             f"RABBITMQ_ERLANG_COOKIE={cookie}",
+            "-i",
             rabbitmq_id,
             "rabbitmq-diagnostics",
             "environment",
@@ -369,9 +370,12 @@ async def nats_connect_ssl(nats_port, user, password, ssl_ctx=None):
 
 def enable_consistent_hash_plugin(rabbitmq_id, cookie):
     p = subprocess.Popen(
-        docker_exec(
+        (
+            "docker",
+            "exec",
             "-e",
             f"RABBITMQ_ERLANG_COOKIE={cookie}",
+            "-i",
             rabbitmq_id,
             "rabbitmq-plugins",
             "enable",
@@ -389,10 +393,10 @@ def get_instances_dir(name):
     run_id = os.environ.get("INTEGRATION_TESTS_RUN_ID", "")
 
     if name:
-        instances_dir_name += "-" + name
+        instances_dir_name += "_" + name
 
     if run_id:
-        instances_dir_name += "-" + shlex.quote(run_id)
+        instances_dir_name += "_" + shlex.quote(run_id)
 
     return instances_dir_name
 
@@ -479,8 +483,8 @@ class ClickHouseCluster:
         self.instances_dir_name = get_instances_dir(self.name)
         xdist_worker = os.getenv("PYTEST_XDIST_WORKER")
         if xdist_worker:
-            self.project_name += f"-{xdist_worker}"
+            self.project_name += f"_{xdist_worker}"
-            self.instances_dir_name += f"-{xdist_worker}"
+            self.instances_dir_name += f"_{xdist_worker}"
 
         self.instances_dir = p.join(self.base_dir, self.instances_dir_name)
         self.docker_logs_path = p.join(self.instances_dir, "docker.log")
@@ -501,7 +505,7 @@ class ClickHouseCluster:
         self.docker_api_version = os.environ.get("DOCKER_API_VERSION")
         self.docker_base_tag = os.environ.get("DOCKER_BASE_TAG", "latest")
 
-        self.base_cmd = ["docker", "compose"]
+        self.base_cmd = ["docker-compose"]
         if custom_dockerd_host:
             self.base_cmd += ["--host", custom_dockerd_host]
         self.base_cmd += ["--env-file", self.env_file]
@@ -779,9 +783,6 @@ class ClickHouseCluster:
 
         self.port_pool = PortPoolManager()
 
-    def compose_cmd(self, *args: str) -> List[str]:
-        return ["docker", "compose", "--project-name", self.project_name, *args]
-
     @property
     def kafka_port(self):
         if self._kafka_port:
@@ -930,7 +931,7 @@ class ClickHouseCluster:
             logging.debug("Trying to prune unused volumes...")
 
             result = run_and_check(["docker volume ls | wc -l"], shell=True)
-            if int(result) > 1:
+            if int(result > 0):
                 run_and_check(["docker", "volume", "prune", "-f"])
             logging.debug(f"Volumes pruned: {result}")
         except:
@@ -956,10 +957,10 @@ class ClickHouseCluster:
     # Returns the list of currently running docker containers corresponding to this ClickHouseCluster.
     def get_running_containers(self):
         # docker-compose names containers using the following formula:
-        # container_name = project_name + '-' + instance_name + '-1'
+        # container_name = project_name + '_' + instance_name + '_1'
         # We need to have "^/" and "$" in the "--filter name" option below to filter by exact name of the container, see
         # https://stackoverflow.com/questions/48767760/how-to-make-docker-container-ls-f-name-filter-by-exact-name
-        filter_name = f"^/{self.project_name}-.*-1$"
+        filter_name = f"^/{self.project_name}_.*_1$"
         # We want the command "docker container list" to show only containers' ID and their names, separated by colon.
         format = "{{.ID}}:{{.Names}}"
         containers = run_and_check(
@@ -1005,12 +1006,15 @@ class ClickHouseCluster:
 
         self.with_zookeeper_secure = True
         self.base_cmd.extend(["--file", zookeeper_docker_compose_path])
-        self.base_zookeeper_cmd = self.compose_cmd(
+        self.base_zookeeper_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             zookeeper_docker_compose_path,
-        )
+        ]
         return self.base_zookeeper_cmd
 
     def setup_zookeeper_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1034,12 +1038,15 @@ class ClickHouseCluster:
 
         self.with_zookeeper = True
         self.base_cmd.extend(["--file", zookeeper_docker_compose_path])
-        self.base_zookeeper_cmd = self.compose_cmd(
+        self.base_zookeeper_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             zookeeper_docker_compose_path,
-        )
+        ]
         return self.base_zookeeper_cmd
 
     def setup_keeper_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1079,12 +1086,15 @@ class ClickHouseCluster:
 
         self.with_zookeeper = True
         self.base_cmd.extend(["--file", keeper_docker_compose_path])
-        self.base_zookeeper_cmd = self.compose_cmd(
+        self.base_zookeeper_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             keeper_docker_compose_path,
-        )
+        ]
         return self.base_zookeeper_cmd
 
     def setup_mysql_client_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1095,12 +1105,15 @@ class ClickHouseCluster:
                 p.join(docker_compose_yml_dir, "docker_compose_mysql_client.yml"),
             ]
         )
-        self.base_mysql_client_cmd = self.compose_cmd(
+        self.base_mysql_client_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_mysql_client.yml"),
-        )
+        ]
 
         return self.base_mysql_client_cmd
 
@@ -1116,12 +1129,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_mysql.yml")]
         )
-        self.base_mysql57_cmd = self.compose_cmd(
+        self.base_mysql57_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_mysql.yml"),
-        )
+        ]
 
         return self.base_mysql57_cmd
 
@@ -1137,12 +1153,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_mysql_8_0.yml")]
         )
-        self.base_mysql8_cmd = self.compose_cmd(
+        self.base_mysql8_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_mysql_8_0.yml"),
-        )
+        ]
 
         return self.base_mysql8_cmd
 
@@ -1160,12 +1179,15 @@ class ClickHouseCluster:
                 p.join(docker_compose_yml_dir, "docker_compose_mysql_cluster.yml"),
             ]
         )
-        self.base_mysql_cluster_cmd = self.compose_cmd(
+        self.base_mysql_cluster_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_mysql_cluster.yml"),
-        )
+        ]
 
         return self.base_mysql_cluster_cmd
 
@@ -1178,12 +1200,15 @@ class ClickHouseCluster:
         env_variables["POSTGRES_LOGS_FS"] = "bind"
 
         self.with_postgres = True
-        self.base_postgres_cmd = self.compose_cmd(
+        self.base_postgres_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_postgres.yml"),
-        )
+        ]
         return self.base_postgres_cmd
 
     def setup_postgres_cluster_cmd(
@@ -1201,12 +1226,15 @@ class ClickHouseCluster:
                 p.join(docker_compose_yml_dir, "docker_compose_postgres_cluster.yml"),
             ]
         )
-        self.base_postgres_cluster_cmd = self.compose_cmd(
+        self.base_postgres_cluster_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_postgres_cluster.yml"),
-        )
+        ]
 
     def setup_postgresql_java_client_cmd(
         self, instance, env_variables, docker_compose_yml_dir
@@ -1220,12 +1248,15 @@ class ClickHouseCluster:
                 ),
             ]
         )
-        self.base_postgresql_java_client_cmd = self.compose_cmd(
+        self.base_postgresql_java_client_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_postgresql_java_client.yml"),
-        )
+        ]
 
     def setup_hdfs_cmd(self, instance, env_variables, docker_compose_yml_dir):
         self.with_hdfs = True
@@ -1237,12 +1268,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_hdfs.yml")]
         )
-        self.base_hdfs_cmd = self.compose_cmd(
+        self.base_hdfs_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_hdfs.yml"),
-        )
+        ]
         logging.debug("HDFS BASE CMD:{self.base_hdfs_cmd)}")
         return self.base_hdfs_cmd
 
@@ -1262,12 +1296,15 @@ class ClickHouseCluster:
                 p.join(docker_compose_yml_dir, "docker_compose_kerberized_hdfs.yml"),
             ]
         )
-        self.base_kerberized_hdfs_cmd = self.compose_cmd(
+        self.base_kerberized_hdfs_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_kerberized_hdfs.yml"),
-        )
+        ]
         return self.base_kerberized_hdfs_cmd
 
     def setup_kafka_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1282,12 +1319,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_kafka.yml")]
         )
-        self.base_kafka_cmd = self.compose_cmd(
+        self.base_kafka_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_kafka.yml"),
-        )
+        ]
         return self.base_kafka_cmd
 
     def setup_kerberized_kafka_cmd(
@@ -1305,12 +1345,15 @@ class ClickHouseCluster:
                 p.join(docker_compose_yml_dir, "docker_compose_kerberized_kafka.yml"),
             ]
         )
-        self.base_kerberized_kafka_cmd = self.compose_cmd(
+        self.base_kerberized_kafka_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_kerberized_kafka.yml"),
-        )
+        ]
         return self.base_kerberized_kafka_cmd
 
     def setup_kerberos_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1323,12 +1366,15 @@ class ClickHouseCluster:
                 p.join(docker_compose_yml_dir, "docker_compose_kerberos_kdc.yml"),
             ]
         )
-        self.base_kerberos_kdc_cmd = self.compose_cmd(
+        self.base_kerberos_kdc_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_kerberos_kdc.yml"),
-        )
+        ]
         return self.base_kerberos_kdc_cmd
 
     def setup_redis_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1340,12 +1386,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_redis.yml")]
         )
-        self.base_redis_cmd = self.compose_cmd(
+        self.base_redis_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_redis.yml"),
-        )
+        ]
         return self.base_redis_cmd
 
     def setup_rabbitmq_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1361,12 +1410,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_rabbitmq.yml")]
         )
-        self.base_rabbitmq_cmd = self.compose_cmd(
+        self.base_rabbitmq_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_rabbitmq.yml"),
-        )
+        ]
         return self.base_rabbitmq_cmd
 
     def setup_nats_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1379,12 +1431,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_nats.yml")]
         )
-        self.base_nats_cmd = self.compose_cmd(
+        self.base_nats_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_nats.yml"),
-        )
+        ]
         return self.base_nats_cmd
 
     def setup_mongo_secure_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1420,12 +1475,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_mongo.yml")]
         )
-        self.base_mongo_cmd = self.compose_cmd(
+        self.base_mongo_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_mongo.yml"),
-        )
+        ]
         return self.base_mongo_cmd
 
     def setup_coredns_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1435,12 +1493,15 @@ class ClickHouseCluster:
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_coredns.yml")]
         )
 
-        self.base_coredns_cmd = self.compose_cmd(
+        self.base_coredns_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_coredns.yml"),
-        )
+        ]
 
         return self.base_coredns_cmd
 
@@ -1455,12 +1516,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_minio.yml")]
         )
-        self.base_minio_cmd = self.compose_cmd(
+        self.base_minio_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_minio.yml"),
-        )
+        ]
         return self.base_minio_cmd
 
     def setup_azurite_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1478,12 +1542,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_azurite.yml")]
         )
-        self.base_azurite_cmd = self.compose_cmd(
+        self.base_azurite_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_azurite.yml"),
-        )
+        ]
         return self.base_azurite_cmd
 
     def setup_cassandra_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1492,12 +1559,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_cassandra.yml")]
         )
-        self.base_cassandra_cmd = self.compose_cmd(
+        self.base_cassandra_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_cassandra.yml"),
-        )
+        ]
         return self.base_cassandra_cmd
 
     def setup_ldap_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1506,12 +1576,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_ldap.yml")]
         )
-        self.base_ldap_cmd = self.compose_cmd(
+        self.base_ldap_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_ldap.yml"),
-        )
+        ]
         return self.base_ldap_cmd
 
     def setup_jdbc_bridge_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1521,12 +1594,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_jdbc_bridge.yml")]
         )
-        self.base_jdbc_bridge_cmd = self.compose_cmd(
+        self.base_jdbc_bridge_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_jdbc_bridge.yml"),
-        )
+        ]
         return self.base_jdbc_bridge_cmd
 
     def setup_nginx_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1535,12 +1611,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_nginx.yml")]
         )
-        self.base_nginx_cmd = self.compose_cmd(
+        self.base_nginx_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_nginx.yml"),
-        )
+        ]
         return self.base_nginx_cmd
 
     def setup_hive(self, instance, env_variables, docker_compose_yml_dir):
@@ -1548,12 +1627,15 @@ class ClickHouseCluster:
         self.base_cmd.extend(
             ["--file", p.join(docker_compose_yml_dir, "docker_compose_hive.yml")]
         )
-        self.base_hive_cmd = self.compose_cmd(
+        self.base_hive_cmd = [
+            "docker-compose",
             "--env-file",
             instance.env_file,
+            "--project-name",
+            self.project_name,
             "--file",
             p.join(docker_compose_yml_dir, "docker_compose_hive.yml"),
-        )
+        ]
         return self.base_hive_cmd
 
     def setup_prometheus_cmd(self, instance, env_variables, docker_compose_yml_dir):
@@ -1581,12 +1663,15 @@ class ClickHouseCluster:
                     p.join(docker_compose_yml_dir, "docker_compose_prometheus.yml"),
                 ]
             )
-            self.base_prometheus_cmd = self.compose_cmd(
+            self.base_prometheus_cmd = [
+                "docker-compose",
                 "--env-file",
                 instance.env_file,
+                "--project-name",
+                self.project_name,
                 "--file",
                 p.join(docker_compose_yml_dir, "docker_compose_prometheus.yml"),
-            )
+            ]
         return self.base_prometheus_cmd
 
     def add_instance(
@@ -1761,15 +1846,13 @@ class ClickHouseCluster:
         )
 
         docker_compose_yml_dir = get_docker_compose_path()
-        docker_compose_net = p.join(docker_compose_yml_dir, "docker_compose_net.yml")
 
         self.instances[name] = instance
-        if not self.with_net_trics and (
+        if ipv4_address is not None or ipv6_address is not None:
-            ipv4_address is not None or ipv6_address is not None
-        ):
-            # docker compose v2 does not accept more than one argument `-f net.yml`
             self.with_net_trics = True
-            self.base_cmd.extend(["--file", docker_compose_net])
+            self.base_cmd.extend(
+                ["--file", p.join(docker_compose_yml_dir, "docker_compose_net.yml")]
+            )
 
         self.base_cmd.extend(["--file", instance.docker_compose_path])
 
@@ -1919,6 +2002,12 @@ class ClickHouseCluster:
                 self.setup_coredns_cmd(instance, env_variables, docker_compose_yml_dir)
             )
 
+        if self.with_net_trics:
+            for cmd in cmds:
+                cmd.extend(
+                    ["--file", p.join(docker_compose_yml_dir, "docker_compose_net.yml")]
+                )
+
         if with_redis and not self.with_redis:
             cmds.append(
                 self.setup_redis_cmd(instance, env_variables, docker_compose_yml_dir)
@@ -1981,13 +2070,6 @@ class ClickHouseCluster:
                 )
             )
 
-        ### !!!! This is the last step after combining all cmds, don't put anything after
-        if self.with_net_trics:
-            for cmd in cmds:
-                # Again, adding it only once
-                if docker_compose_net not in cmd:
-                    cmd.extend(["--file", docker_compose_net])
-
         logging.debug(
             "Cluster name:{} project_name:{}. Added instance name:{} tag:{} base_cmd:{} docker_compose_yml_dir:{}".format(
                 self.name,
@@ -2002,7 +2084,7 @@ class ClickHouseCluster:
 
     def get_instance_docker_id(self, instance_name):
         # According to how docker-compose names containers.
-        return self.project_name + "-" + instance_name + "-1"
+        return self.project_name + "_" + instance_name + "_1"
 
     def _replace(self, path, what, to):
         with open(path, "r") as p:
@@ -2741,7 +2823,7 @@ class ClickHouseCluster:
                         "Got exception pulling images: %s", kwargs["exception"]
                     )
 
-            retry(log_function=logging_pulling_images)(run_and_check, images_pull_cmd)
+            retry(log_function=logging_pulling_images)(run_and_check)(images_pull_cmd)
 
             if self.with_zookeeper_secure and self.base_zookeeper_cmd:
                 logging.debug("Setup ZooKeeper Secure")
@@ -3022,7 +3104,9 @@ class ClickHouseCluster:
 
                 retry(
                     log_function=logging_azurite_initialization,
-                )(run_and_check, azurite_start_cmd)
+                )(
+                    run_and_check
+                )(azurite_start_cmd)
                 self.up_called = True
                 logging.info("Trying to connect to Azurite")
                 self.wait_azurite_to_start()
@@ -3153,7 +3237,7 @@ class ClickHouseCluster:
                 )
         else:
             logging.warning(
-                "docker compose up was not called. Trying to export docker.log for running containers"
+                "docker-compose up was not called. Trying to export docker.log for running containers"
             )
 
         self.cleanup()
@@ -3240,7 +3324,8 @@ class ClickHouseCluster:
             subprocess_check_call(self.base_zookeeper_cmd + ["start", n])
 
 
-DOCKER_COMPOSE_TEMPLATE = """---
+DOCKER_COMPOSE_TEMPLATE = """
+version: '2.3'
 services:
     {name}:
         image: {image}:{tag}

tests/integration/helpers/keeper_utils.py

@@ -1,46 +1,13 @@
-import contextlib
 import io
-import re
-import select
-import socket
 import subprocess
+import socket
 import time
 import typing as tp
-
+import contextlib
-from helpers.client import CommandRequest
+import select
-from helpers.cluster import ClickHouseCluster, ClickHouseInstance
 from kazoo.client import KazooClient
-
+from helpers.cluster import ClickHouseCluster, ClickHouseInstance
-ss_established = [
+from helpers.client import CommandRequest
-    "ss",
-    "--resolve",
-    "--tcp",
-    "--no-header",
-    "state",
-    "ESTABLISHED",
-    "( dport = 2181 or sport = 2181 )",
-]
-
-
-def get_active_zk_connections(node: ClickHouseInstance) -> tp.List[str]:
-    return (
-        str(node.exec_in_container(ss_established, privileged=True, user="root"))
-        .strip()
-        .split("\n")
-    )
-
-
-def get_zookeeper_which_node_connected_to(node: ClickHouseInstance) -> str:
-    line = str(
-        node.exec_in_container(ss_established, privileged=True, user="root")
-    ).strip()
-
-    pattern = re.compile(r"zoo[0-9]+", re.IGNORECASE)
-    result = pattern.findall(line)
-    assert (
-        len(result) == 1
-    ), "ClickHouse must be connected only to one Zookeeper at a time"
-    return result[0]
 
 
 def execute_keeper_client_query(

tests/integration/helpers/retry_decorator.py

@@ -4,31 +4,36 @@ from typing import Type, List
 
 
 def retry(
-    *exceptions: Type[BaseException],
     retries: int = 5,
     delay: float = 1,
     backoff: float = 1.5,
     jitter: float = 2,
     log_function=None,  # should take **kwargs or arguments: `retry_number`, `exception` and `sleep_time`
+    retriable_expections_list: List[Type[BaseException]] = [Exception],
 ):
-    exceptions = exceptions or (Exception,)
+    def inner(func):
+        def wrapper(*args, **kwargs):
+            current_delay = delay
+            for retry in range(retries):
+                try:
+                    func(*args, **kwargs)
+                    break
+                except Exception as e:
+                    should_retry = False
+                    for retriable_exception in retriable_expections_list:
+                        if isinstance(e, retriable_exception):
+                            should_retry = True
+                            break
+                    if not should_retry or (retry == retries - 1):
+                        raise e
+                    sleep_time = current_delay + random.uniform(0, jitter)
+                    if log_function is not None:
+                        log_function(
+                            retry_number=retry, exception=e, sleep_time=sleep_time
+                        )
+                    time.sleep(sleep_time)
+                    current_delay *= backoff
 
-    def inner(func, *args, **kwargs):
+        return wrapper
-        current_delay = delay
-        for retry in range(retries):
-            try:
-                func(*args, **kwargs)
-                break
-            except Exception as e:
-                should_retry = (retry < retries - 1) and any(
-                    isinstance(e, re) for re in exceptions
-                )
-                if not should_retry:
-                    raise e
-                sleep_time = current_delay + random.uniform(0, jitter)
-                if log_function is not None:
-                    log_function(retry_number=retry, exception=e, sleep_time=sleep_time)
-                time.sleep(sleep_time)
-                current_delay *= backoff
 
     return inner

tests/integration/runner

@@ -424,7 +424,7 @@ if __name__ == "__main__":
     cmd = cmd_base + " " + args.command
     cmd_pre_pull = (
         f"{cmd_base} find /ClickHouse/tests/integration/compose -name docker_compose_*.yml "
-        r"-exec docker compose -f '{}' pull \;"
+        r"-exec docker-compose -f '{}' pull \;"
     )
 
     containers = subprocess.check_output(

tests/integration/test_dotnet_client/test.py

@@ -37,16 +37,19 @@ def dotnet_container():
         DOCKER_COMPOSE_PATH, "docker_compose_dotnet_client.yml"
     )
     run_and_check(
-        cluster.compose_cmd(
+        [
+            "docker-compose",
+            "-p",
+            cluster.project_name,
             "-f",
             docker_compose,
             "up",
             "--force-recreate",
             "-d",
             "--no-build",
-        )
+        ]
     )
-    yield docker.from_env().containers.get(cluster.get_instance_docker_id("dotnet1"))
+    yield docker.from_env().containers.get(cluster.project_name + "_dotnet1_1")
 
 
 def test_dotnet_client(started_cluster, dotnet_container):

tests/integration/test_keeper_s3_snapshot/test.py

@@ -1,14 +1,12 @@
-from multiprocessing.dummy import Pool
-from time import sleep
-
-import helpers.keeper_utils as keeper_utils
 import pytest
-from helpers import keeper_utils
 from helpers.cluster import ClickHouseCluster
-from helpers.retry_decorator import retry
+from time import sleep
-from kazoo.client import KazooClient
-from minio.deleteobjects import DeleteObject
 from retry import retry
+from multiprocessing.dummy import Pool
+import helpers.keeper_utils as keeper_utils
+from minio.deleteobjects import DeleteObject
+
+from kazoo.client import KazooClient
 
 # from kazoo.protocol.serialization import Connect, read_buffer, write_buffer
 
@@ -111,6 +109,7 @@ def test_s3_upload(started_cluster):
             cluster.minio_client.remove_object("snapshots", s.object_name)
 
     # Keeper sends snapshots asynchornously, hence we need to retry.
+    @retry(AssertionError, tries=10, delay=2)
     def _check_snapshots():
         assert set(get_saved_snapshots()) == set(
             [
@@ -121,7 +120,7 @@ def test_s3_upload(started_cluster):
             ]
         )
 
-    retry(AssertionError, retries=10, delay=2, jitter=0, backoff=1)(_check_snapshots)
+    _check_snapshots()
 
     destroy_zk_client(node1_zk)
     node1.stop_clickhouse(kill=True)
@@ -133,13 +132,10 @@ def test_s3_upload(started_cluster):
     for _ in range(200):
         node2_zk.create("/test", sequence=True)
 
+    @retry(AssertionError, tries=10, delay=2)
     def _check_snapshots_without_quorum():
         assert len(get_saved_snapshots()) > 4
 
-    retry(AssertionError, retries=10, delay=2, jitter=0, backoff=1)(
-        _check_snapshots_without_quorum
-    )
-
     _check_snapshots_without_quorum()
 
     success_upload_message = "Successfully uploaded"

tests/integration/test_mysql_protocol/test.py

@@ -55,7 +55,8 @@ def golang_container():
         DOCKER_COMPOSE_PATH, "docker_compose_mysql_golang_client.yml"
     )
     run_and_check(
-        cluster.compose_cmd(
+        [
+            "docker-compose",
             "-p",
             cluster.project_name,
             "-f",
@@ -64,13 +65,13 @@ def golang_container():
             "--force-recreate",
             "-d",
             "--no-build",
-        )
+        ]
     )
     yield docker.DockerClient(
         base_url="unix:///var/run/docker.sock",
         version=cluster.docker_api_version,
         timeout=600,
-    ).containers.get(cluster.get_instance_docker_id("golang1"))
+    ).containers.get(cluster.project_name + "_golang1_1")
 
 
 @pytest.fixture(scope="module")
@@ -79,22 +80,25 @@ def php_container():
         DOCKER_COMPOSE_PATH, "docker_compose_mysql_php_client.yml"
     )
     run_and_check(
-        cluster.compose_cmd(
+        [
+            "docker-compose",
             "--env-file",
             cluster.instances["node"].env_file,
+            "-p",
+            cluster.project_name,
             "-f",
             docker_compose,
             "up",
             "--force-recreate",
             "-d",
             "--no-build",
-        )
+        ]
     )
     yield docker.DockerClient(
         base_url="unix:///var/run/docker.sock",
         version=cluster.docker_api_version,
         timeout=600,
-    ).containers.get(cluster.get_instance_docker_id("php1"))
+    ).containers.get(cluster.project_name + "_php1_1")
 
 
 @pytest.fixture(scope="module")
@@ -103,22 +107,25 @@ def nodejs_container():
         DOCKER_COMPOSE_PATH, "docker_compose_mysql_js_client.yml"
     )
     run_and_check(
-        cluster.compose_cmd(
+        [
+            "docker-compose",
             "--env-file",
             cluster.instances["node"].env_file,
+            "-p",
+            cluster.project_name,
             "-f",
             docker_compose,
             "up",
             "--force-recreate",
             "-d",
             "--no-build",
-        )
+        ]
     )
     yield docker.DockerClient(
         base_url="unix:///var/run/docker.sock",
         version=cluster.docker_api_version,
         timeout=600,
-    ).containers.get(cluster.get_instance_docker_id("mysqljs1"))
+    ).containers.get(cluster.project_name + "_mysqljs1_1")
 
 
 @pytest.fixture(scope="module")
@@ -127,22 +134,25 @@ def java_container():
         DOCKER_COMPOSE_PATH, "docker_compose_mysql_java_client.yml"
     )
     run_and_check(
-        cluster.compose_cmd(
+        [
+            "docker-compose",
             "--env-file",
             cluster.instances["node"].env_file,
+            "-p",
+            cluster.project_name,
             "-f",
             docker_compose,
             "up",
             "--force-recreate",
             "-d",
             "--no-build",
-        )
+        ]
     )
     yield docker.DockerClient(
         base_url="unix:///var/run/docker.sock",
         version=cluster.docker_api_version,
         timeout=600,
-    ).containers.get(cluster.get_instance_docker_id("java1"))
+    ).containers.get(cluster.project_name + "_java1_1")
 
 
 def test_mysql_client(started_cluster):

tests/integration/test_read_only_table/test.py

@@ -1,10 +1,9 @@
-import logging
-import re
 import time
+import re
+import logging
 
 import pytest
 from helpers.cluster import ClickHouseCluster
-from helpers.keeper_utils import get_zookeeper_which_node_connected_to
 from helpers.test_tools import assert_eq_with_retry
 
 NUM_TABLES = 10
@@ -57,6 +56,26 @@ def test_restart_zookeeper(start_cluster):
 
     logging.info("Inserted test data and initialized all tables")
 
+    def get_zookeeper_which_node_connected_to(node):
+        line = str(
+            node.exec_in_container(
+                [
+                    "bash",
+                    "-c",
+                    "lsof -a -i4 -i6 -itcp -w | grep 2181 | grep ESTABLISHED",
+                ],
+                privileged=True,
+                user="root",
+            )
+        ).strip()
+
+        pattern = re.compile(r"zoo[0-9]+", re.IGNORECASE)
+        result = pattern.findall(line)
+        assert (
+            len(result) == 1
+        ), "ClickHouse must be connected only to one Zookeeper at a time"
+        return result[0]
+
     node1_zk = get_zookeeper_which_node_connected_to(node1)
 
     # ClickHouse should +- immediately reconnect to another zookeeper node

tests/integration/test_reload_zookeeper/test.py

@@ -1,12 +1,12 @@
-import os
 import time
-
 import pytest
-from helpers.client import QueryRuntimeException
+import os
+
 from helpers.cluster import ClickHouseCluster
-from helpers.keeper_utils import get_active_zk_connections
+from helpers.client import QueryRuntimeException
 from helpers.test_tools import assert_eq_with_retry
 
+
 cluster = ClickHouseCluster(__file__, zookeeper_config_path="configs/zookeeper.xml")
 node = cluster.add_instance("node", with_zookeeper=True)
 
@@ -85,6 +85,19 @@ def test_reload_zookeeper(start_cluster):
             settings={"select_sequential_consistency": 1},
         )
 
+    def get_active_zk_connections():
+        return str(
+            node.exec_in_container(
+                [
+                    "bash",
+                    "-c",
+                    "lsof -a -i4 -i6 -itcp -w | grep 2181 | grep ESTABLISHED | wc -l",
+                ],
+                privileged=True,
+                user="root",
+            )
+        ).strip()
+
     ## set config to zoo2, server will be normal
     new_config = """
 <clickhouse>
@@ -100,16 +113,16 @@ def test_reload_zookeeper(start_cluster):
     node.replace_config("/etc/clickhouse-server/conf.d/zookeeper.xml", new_config)
     node.query("SYSTEM RELOAD CONFIG")
 
-    active_zk_connections = get_active_zk_connections(node)
+    active_zk_connections = get_active_zk_connections()
     assert (
-        len(active_zk_connections) == 1
+        active_zk_connections == "1"
     ), "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections)
 
     assert_eq_with_retry(
         node, "SELECT COUNT() FROM test_table", "1000", retry_count=120, sleep_time=0.5
     )
 
-    active_zk_connections = get_active_zk_connections(node)
+    active_zk_connections = get_active_zk_connections()
     assert (
-        len(active_zk_connections) == 1
+        active_zk_connections == "1"
     ), "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections)

tests/integration/test_replicated_user_defined_functions/test.py

@@ -1,13 +1,13 @@
 import inspect
-import os.path
-import time
 from contextlib import nullcontext as does_not_raise
 
 import pytest
-from helpers.client import QueryRuntimeException
+import time
+import os.path
+
 from helpers.cluster import ClickHouseCluster
-from helpers.keeper_utils import get_active_zk_connections
+from helpers.client import QueryRuntimeException
-from helpers.test_tools import TSV, assert_eq_with_retry
+from helpers.test_tools import assert_eq_with_retry, TSV
 
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 
@@ -65,6 +65,20 @@ def revert_zookeeper_config():
         replace_zookeeper_config(f.read())
 
 
+def get_active_zk_connections():
+    return str(
+        node1.exec_in_container(
+            [
+                "bash",
+                "-c",
+                "lsof -a -i4 -i6 -itcp -w | grep 2181 | grep ESTABLISHED | wc -l",
+            ],
+            privileged=True,
+            user="root",
+        )
+    ).strip()
+
+
 def test_create_and_drop():
     node1.query("CREATE FUNCTION f1 AS (x, y) -> x + y")
     assert node1.query("SELECT f1(12, 3)") == "15\n"
@@ -252,9 +266,9 @@ def test_reload_zookeeper():
         )
     )
 
-    active_zk_connections = get_active_zk_connections(node1)
+    active_zk_connections = get_active_zk_connections()
     assert (
-        len(active_zk_connections) == 1
+        active_zk_connections == "1"
     ), "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections)
 
     node1.query("CREATE FUNCTION f3 AS (x, y) -> x / y")
@@ -266,9 +280,9 @@ def test_reload_zookeeper():
 
     assert node2.query("SELECT f1(12, 3), f2(), f3(12, 3)") == TSV([[15, 2, 4]])
 
-    active_zk_connections = get_active_zk_connections(node1)
+    active_zk_connections = get_active_zk_connections()
     assert (
-        len(active_zk_connections) == 1
+        active_zk_connections == "1"
     ), "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections)
 
     node1.query("DROP FUNCTION f1")

tests/integration/test_replicated_users/test.py

@@ -1,11 +1,10 @@
 import inspect
-import time
-from dataclasses import dataclass
-
 import pytest
+import time
+
+from dataclasses import dataclass
 from helpers.cluster import ClickHouseCluster
-from helpers.keeper_utils import get_active_zk_connections
+from helpers.test_tools import assert_eq_with_retry, TSV
-from helpers.test_tools import TSV, assert_eq_with_retry
 
 cluster = ClickHouseCluster(__file__, zookeeper_config_path="configs/zookeeper.xml")
 
@@ -189,6 +188,19 @@ def test_reload_zookeeper(started_cluster):
         node1.query("SYSTEM RELOAD CONFIG")
         node2.query("SYSTEM RELOAD CONFIG")
 
+    def get_active_zk_connections():
+        return str(
+            node1.exec_in_container(
+                [
+                    "bash",
+                    "-c",
+                    "lsof -a -i4 -i6 -itcp -w | grep 2181 | grep ESTABLISHED | wc -l",
+                ],
+                privileged=True,
+                user="root",
+            )
+        ).strip()
+
     node1.query("CREATE USER u1")
     assert_eq_with_retry(
         node2, "SELECT name FROM system.users WHERE name ='u1'", "u1\n"
@@ -247,9 +259,9 @@ def test_reload_zookeeper(started_cluster):
 """
     )
 
-    active_zk_connections = get_active_zk_connections(node1)
+    active_zk_connections = get_active_zk_connections()
     assert (
-        len(active_zk_connections) == 1
+        active_zk_connections == "1"
     ), "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections)
 
     node1.query("CREATE USER u3")
@@ -259,7 +271,7 @@ def test_reload_zookeeper(started_cluster):
         TSV(["u1", "u2", "u3"]),
     )
 
-    active_zk_connections = get_active_zk_connections(node1)
+    active_zk_connections = get_active_zk_connections()
     assert (
-        len(active_zk_connections) == 1
+        active_zk_connections == "1"
     ), "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections)

tests/integration/test_zookeeper_config_load_balancing/test.py

@@ -1,9 +1,6 @@
-import logging
-import re
 import time
 import pytest
 from helpers.cluster import ClickHouseCluster
-from helpers.keeper_utils import ss_established
 from helpers.network import PartitionManager
 from helpers.test_tools import assert_eq_with_retry
 
@@ -21,23 +18,11 @@ node2 = cluster.add_instance(
 node3 = cluster.add_instance(
     "nod3", with_zookeeper=True, main_configs=["configs/zookeeper_load_balancing.xml"]
 )
+
 node4 = cluster.add_instance(
     "nod4", with_zookeeper=True, main_configs=["configs/zookeeper_load_balancing2.xml"]
 )
 
-zk1_re = re.compile(
-    r"testzookeeperconfigloadbalancing-(gw\d+-)?zoo1-1"
-    r".*testzookeeperconfigloadbalancing(-gw\d+)?_default:2181"
-)
-zk2_re = re.compile(
-    r"testzookeeperconfigloadbalancing-(gw\d+-)?zoo2-1"
-    r".*testzookeeperconfigloadbalancing(-gw\d+)?_default:2181"
-)
-zk3_re = re.compile(
-    r"testzookeeperconfigloadbalancing-(gw\d+-)?zoo3-1"
-    r".*testzookeeperconfigloadbalancing(-gw\d+)?_default:2181"
-)
-
 
 def change_balancing(old, new, reload=True):
     line = "<zookeeper_load_balancing>{}<"
@@ -66,15 +51,89 @@ def started_cluster():
 def test_first_or_random(started_cluster):
     try:
         change_balancing("random", "first_or_random")
-        for node in (node1, node2, node3):
+        print(
-            connections = (
+            str(
-                node.exec_in_container(ss_established, privileged=True, user="root")
+                node1.exec_in_container(
-                .strip()
+                    [
-                .split("\n")
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
             )
-            logging.debug("Established connections for 2181:\n%s", connections)
+        )
-            assert len(connections) == 1
+        assert (
-            assert zk1_re.search(connections[0])
+            "1"
+            == str(
+                node1.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            ).strip()
+        )
+
+        print(
+            str(
+                node2.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            )
+        )
+        assert (
+            "1"
+            == str(
+                node2.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            ).strip()
+        )
+
+        print(
+            str(
+                node3.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            )
+        )
+        assert (
+            "1"
+            == str(
+                node3.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            ).strip()
+        )
     finally:
         change_balancing("first_or_random", "random", reload=False)
 
@@ -82,15 +141,89 @@ def test_first_or_random(started_cluster):
 def test_in_order(started_cluster):
     try:
         change_balancing("random", "in_order")
-        for node in (node1, node2, node3):
+        print(
-            connections = (
+            str(
-                node.exec_in_container(ss_established, privileged=True, user="root")
+                node1.exec_in_container(
-                .strip()
+                    [
-                .split("\n")
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
             )
-            logging.debug("Established connections for 2181:\n%s", connections)
+        )
-            assert len(connections) == 1
+        assert (
-            assert zk1_re.search(connections[0])
+            "1"
+            == str(
+                node1.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            ).strip()
+        )
+
+        print(
+            str(
+                node2.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            )
+        )
+        assert (
+            "1"
+            == str(
+                node2.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            ).strip()
+        )
+
+        print(
+            str(
+                node3.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            )
+        )
+        assert (
+            "1"
+            == str(
+                node3.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            ).strip()
+        )
     finally:
         change_balancing("in_order", "random", reload=False)
 
@@ -98,15 +231,89 @@ def test_in_order(started_cluster):
 def test_nearest_hostname(started_cluster):
     try:
         change_balancing("random", "nearest_hostname")
-        for node, regexp in ((node1, zk1_re), (node2, zk2_re), (node3, zk3_re)):
+        print(
-            connections = (
+            str(
-                node.exec_in_container(ss_established, privileged=True, user="root")
+                node1.exec_in_container(
-                .strip()
+                    [
-                .split("\n")
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
             )
-            logging.debug("Established connections for 2181:\n%s", connections)
+        )
-            assert len(connections) == 1
+        assert (
-            assert regexp.search(connections[0])
+            "1"
+            == str(
+                node1.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            ).strip()
+        )
+
+        print(
+            str(
+                node2.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            )
+        )
+        assert (
+            "1"
+            == str(
+                node2.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo2_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            ).strip()
+        )
+
+        print(
+            str(
+                node3.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            )
+        )
+        assert (
+            "1"
+            == str(
+                node3.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo3_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            ).strip()
+        )
     finally:
         change_balancing("nearest_hostname", "random", reload=False)
 
@@ -114,15 +321,89 @@ def test_nearest_hostname(started_cluster):
 def test_hostname_levenshtein_distance(started_cluster):
     try:
         change_balancing("random", "hostname_levenshtein_distance")
-        for node, regexp in ((node1, zk1_re), (node2, zk2_re), (node3, zk3_re)):
+        print(
-            connections = (
+            str(
-                node.exec_in_container(ss_established, privileged=True, user="root")
+                node1.exec_in_container(
-                .strip()
+                    [
-                .split("\n")
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
             )
-            logging.debug("Established connections for 2181:\n%s", connections)
+        )
-            assert len(connections) == 1
+        assert (
-            assert regexp.search(connections[0])
+            "1"
+            == str(
+                node1.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            ).strip()
+        )
+
+        print(
+            str(
+                node2.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            )
+        )
+        assert (
+            "1"
+            == str(
+                node2.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo2_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            ).strip()
+        )
+
+        print(
+            str(
+                node3.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            )
+        )
+        assert (
+            "1"
+            == str(
+                node3.exec_in_container(
+                    [
+                        "bash",
+                        "-c",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo3_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
+                    ],
+                    privileged=True,
+                    user="root",
+                )
+            ).strip()
+        )
     finally:
         change_balancing("hostname_levenshtein_distance", "random", reload=False)