ClickHouse/tests/ci/docker_pull_helper.py

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

81 lines
2.4 KiB
Python
Raw Normal View History

2021-11-12 12:13:13 +00:00
#!/usr/bin/env python3
import os
import json
import time
import subprocess
import logging
from typing import Optional
2021-11-12 12:13:13 +00:00
class DockerImage:
def __init__(self, name, version: Optional[str] = None):
2021-11-12 12:13:13 +00:00
self.name = name
if version is None:
self.version = "latest"
2021-11-12 12:13:13 +00:00
else:
self.version = version
def __str__(self):
return f"{self.name}:{self.version}"
def get_images_with_versions(
reports_path, required_image, pull=True, version: Optional[str] = None
):
2021-11-12 12:36:25 +00:00
images_path = None
2021-11-12 12:13:13 +00:00
for root, _, files in os.walk(reports_path):
for f in files:
if f == "changed_images.json":
images_path = os.path.join(root, "changed_images.json")
2021-11-12 12:13:13 +00:00
break
2021-12-10 07:54:37 +00:00
if not images_path:
logging.info("Images file not found")
else:
logging.info("Images file path %s", images_path)
2021-11-12 12:36:25 +00:00
if images_path is not None and os.path.exists(images_path):
logging.info("Images file exists")
with open(images_path, "r", encoding="utf-8") as images_fd:
2021-11-12 12:36:25 +00:00
images = json.load(images_fd)
logging.info("Got images %s", images)
2021-11-12 12:13:13 +00:00
else:
2021-11-12 12:36:25 +00:00
images = {}
docker_images = []
2021-11-12 12:50:56 +00:00
for image_name in required_image:
docker_image = DockerImage(image_name, version)
2021-11-12 12:36:25 +00:00
if image_name in images:
docker_image.version = images[image_name]
docker_images.append(docker_image)
2021-11-12 12:13:13 +00:00
if pull:
for docker_image in docker_images:
for i in range(10):
try:
logging.info("Pulling image %s", docker_image)
subprocess.check_output(
f"docker pull {docker_image}",
stderr=subprocess.STDOUT,
shell=True,
)
2021-11-12 12:13:13 +00:00
break
except Exception as ex:
latest_error = ex
2021-11-12 12:13:13 +00:00
time.sleep(i * 3)
logging.info("Got execption pulling docker %s", ex)
else:
raise Exception(
"Cannot pull dockerhub for image docker pull "
f"{docker_image} because of {latest_error}"
)
2021-11-12 12:13:13 +00:00
return docker_images
2022-03-08 15:19:05 +00:00
def get_image_with_version(reports_path, image, pull=True, version=None):
2022-04-13 11:45:54 +00:00
logging.info("Looking for images file in %s", reports_path)
2022-03-08 15:19:05 +00:00
return get_images_with_versions(reports_path, [image], pull, version=version)[0]