thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14112 from vzakaznikov/testflows_increase_compose_http_timeout_and_retry_docker_compose_up

Browse Source 
testflows: adding retry logic when bringing up docker-compose cluster
This commit is contained in:
alexey-milovidov 2020-08-27 15:51:16 +03:00 committed by GitHub
commit e1cf919c93
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 25 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docker/test/testflows/runner/Dockerfile
View File

@ -35,7 +35,7 @@ RUN apt-get update \
ENV TZ=Europe/Moscow
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN pip3 install urllib3 testflows==1.6.39 docker-compose docker dicttoxml kazoo tzlocal
RUN pip3 install urllib3 testflows==1.6.42 docker-compose docker dicttoxml kazoo tzlocal
ENV DOCKER_CHANNEL stable
ENV DOCKER_VERSION 17.09.1-ce

35
tests/testflows/helpers/cluster.py
View File

@ -246,6 +246,7 @@ class Cluster(object):
assert os.path.exists(self.clickhouse_binary_path)
with And("I set all the necessary environment variables"):
os.environ["COMPOSE_HTTP_TIMEOUT"] = "300"
os.environ["CLICKHOUSE_TESTS_SERVER_BIN_PATH"] = self.clickhouse_binary_path
os.environ["CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH"] = os.path.join(
os.path.dirname(self.clickhouse_binary_path), "clickhouse-odbc-bridge")
@ -255,18 +256,30 @@ class Cluster(object):
self.command(None, "env | grep CLICKHOUSE")
with Given("docker-compose"):
with By("pulling images for all the services"):
self.command(None, f'{self.docker_compose} pull 2>&1 | tee', exitcode=0, timeout=timeout)
with And("executing docker-compose down just in case it is up"):
self.command(None, f'{self.docker_compose} down 2>&1 | tee', exitcode=0, timeout=timeout)
with And("executing docker-compose up"):
cmd = self.command(None, f'{self.docker_compose} up -d 2>&1 | tee', timeout=timeout)
max_attempts = 5
for attempt in range(max_attempts):
with When(f"attempt {attempt}/{max_attempts}"):
with By("pulling images for all the services"):
cmd = self.command(None, f"{self.docker_compose} pull 2>&1 | tee", exitcode=None, timeout=timeout)
if cmd.exitcode != 0:
continue
with And("executing docker-compose down just in case it is up"):
cmd = self.command(None, f"{self.docker_compose} down 2>&1 | tee", exitcode=None, timeout=timeout)
if cmd.exitcode != 0:
continue
with And("executing docker-compose up"):
cmd = self.command(None, f"{self.docker_compose} up -d 2>&1 | tee", timeout=timeout)
with Then("check there are no unhealthy containers"):
if "is unhealthy" in cmd.output:
self.command(None, f'{self.docker_compose} ps | tee')
self.command(None, f'{self.docker_compose} logs | tee')
fail("found unhealthy containers")
with Then("check there are no unhealthy containers"):
if "is unhealthy" in cmd.output:
self.command(None, f"{self.docker_compose} ps | tee")
self.command(None, f"{self.docker_compose} logs | tee")
if cmd.exitcode == 0:
break
if cmd.exitcode != 0:
fail("could not bring up docker-compose cluster")
with Then("wait all nodes report healhy"):
for name in self.nodes["clickhouse"]: