ClickHouse/tests/ci/docker_images_check.py

498 lines
15 KiB
Python
Raw Normal View History

2021-09-15 16:32:17 +00:00
#!/usr/bin/env python3
import argparse
2021-09-15 17:01:16 +00:00
import json
import logging
2021-09-15 16:58:36 +00:00
import os
2022-02-10 16:34:55 +00:00
import platform
import shutil
import subprocess
import time
2022-03-29 12:41:47 +00:00
import sys
from pathlib import Path
2022-11-10 15:57:01 +00:00
from typing import Any, Dict, List, Optional, Set, Tuple, Union
from github import Github
2021-11-26 14:00:09 +00:00
from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
from commit_status_helper import post_commit_status
2022-08-11 13:01:32 +00:00
from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, GITHUB_RUN_URL
2021-10-20 11:48:27 +00:00
from get_robot_token import get_best_robot_token, get_parameter_from_ssm
from pr_info import PRInfo
from report import TestResults, TestResult
from s3_helper import S3Helper
2021-11-19 14:47:04 +00:00
from stopwatch import Stopwatch
from upload_result_helper import upload_results
2021-09-15 16:32:17 +00:00
NAME = "Push to Dockerhub"
2021-09-15 16:32:17 +00:00
TEMP_PATH = os.path.join(RUNNER_TEMP, "docker_images_check")
2022-02-10 16:34:55 +00:00
ImagesDict = Dict[str, dict]
2021-12-21 16:32:04 +00:00
class DockerImage:
def __init__(
self,
path: str,
repo: str,
2022-02-10 16:34:55 +00:00
only_amd64: bool,
parent: Optional["DockerImage"] = None,
gh_repo_path: str = GITHUB_WORKSPACE,
):
self.path = path
self.full_path = os.path.join(gh_repo_path, path)
self.repo = repo
2022-02-10 16:34:55 +00:00
self.only_amd64 = only_amd64
self.parent = parent
self.built = False
def __eq__(self, other) -> bool: # type: ignore
"""Is used to check if DockerImage is in a set or not"""
2022-02-10 16:34:55 +00:00
return (
self.path == other.path
and self.repo == self.repo
and self.only_amd64 == other.only_amd64
)
2022-11-10 15:57:01 +00:00
def __lt__(self, other: Any) -> bool:
if not isinstance(other, DockerImage):
return False
if self.parent and not other.parent:
return False
if not self.parent and other.parent:
return True
if self.path < other.path:
return True
if self.repo < other.repo:
return True
return False
def __hash__(self):
return hash(self.path)
def __str__(self):
return self.repo
def __repr__(self):
return f"DockerImage(path={self.path},repo={self.repo},parent={self.parent})"
def get_images_dict(repo_path: str, image_file_path: str) -> ImagesDict:
2022-02-10 16:34:55 +00:00
"""Return images suppose to build on the current architecture host"""
2021-09-15 16:32:17 +00:00
images_dict = {}
path_to_images_file = os.path.join(repo_path, image_file_path)
if os.path.exists(path_to_images_file):
2022-01-18 21:28:26 +00:00
with open(path_to_images_file, "rb") as dict_file:
2021-09-15 16:32:17 +00:00
images_dict = json.load(dict_file)
else:
2021-12-21 16:32:04 +00:00
logging.info(
2023-01-08 06:08:20 +00:00
"Image file %s doesn't exist in repo %s", image_file_path, repo_path
2021-12-21 16:32:04 +00:00
)
2021-09-15 16:32:17 +00:00
return images_dict
def get_changed_docker_images(
pr_info: PRInfo, images_dict: ImagesDict
) -> Set[DockerImage]:
2021-09-15 16:32:17 +00:00
if not images_dict:
return set()
2021-09-15 16:32:17 +00:00
files_changed = pr_info.changed_files
2021-12-21 16:32:04 +00:00
logging.info(
"Changed files for PR %s @ %s: %s",
pr_info.number,
pr_info.sha,
str(files_changed),
)
2021-09-15 16:32:17 +00:00
changed_images = []
for dockerfile_dir, image_description in images_dict.items():
2021-11-23 09:43:49 +00:00
for f in files_changed:
if f.startswith(dockerfile_dir):
name = image_description["name"]
2022-02-10 16:34:55 +00:00
only_amd64 = image_description.get("only_amd64", False)
2021-11-23 09:43:49 +00:00
logging.info(
2021-12-21 16:32:04 +00:00
"Found changed file '%s' which affects "
"docker image '%s' with path '%s'",
f,
name,
2021-12-21 16:32:04 +00:00
dockerfile_dir,
)
2022-02-10 16:34:55 +00:00
changed_images.append(DockerImage(dockerfile_dir, name, only_amd64))
2021-11-23 09:43:49 +00:00
break
2021-09-15 16:32:17 +00:00
# The order is important: dependents should go later than bases, so that
# they are built with updated base versions.
index = 0
while index < len(changed_images):
image = changed_images[index]
for dependent in images_dict[image.path]["dependent"]:
2021-09-15 16:32:17 +00:00
logging.info(
2021-12-21 16:32:04 +00:00
"Marking docker image '%s' as changed because it "
"depends on changed docker image '%s'",
dependent,
image,
)
2022-02-10 16:34:55 +00:00
name = images_dict[dependent]["name"]
only_amd64 = images_dict[dependent].get("only_amd64", False)
changed_images.append(DockerImage(dependent, name, only_amd64, image))
2021-09-15 16:32:17 +00:00
index += 1
if index > 5 * len(images_dict):
2021-09-15 16:32:17 +00:00
# Sanity check to prevent infinite loop.
2021-12-21 16:32:04 +00:00
raise RuntimeError(
f"Too many changed docker images, this is a bug. {changed_images}"
)
2021-09-15 16:32:17 +00:00
# With reversed changed_images set will use images with parents first, and
# images without parents then
result = set(reversed(changed_images))
2021-12-21 16:32:04 +00:00
logging.info(
"Changed docker images for PR %s @ %s: '%s'",
pr_info.number,
pr_info.sha,
result,
)
return result
2021-09-15 16:32:17 +00:00
2021-12-21 16:32:04 +00:00
def gen_versions(
pr_info: PRInfo, suffix: Optional[str]
) -> Tuple[List[str], Union[str, List[str]]]:
pr_commit_version = str(pr_info.number) + "-" + pr_info.sha
# The order is important, PR number is used as cache during the build
versions = [str(pr_info.number), pr_commit_version]
result_version = pr_commit_version
if pr_info.number == 0 and pr_info.base_ref == "master":
# First get the latest for cache
versions.insert(0, "latest")
if suffix:
# We should build architecture specific images separately and merge a
# manifest lately in a different script
versions = [f"{v}-{suffix}" for v in versions]
# changed_images_{suffix}.json should contain all changed images
result_version = versions
return versions, result_version
2022-02-10 16:34:55 +00:00
def build_and_push_dummy_image(
image: DockerImage,
version_string: str,
push: bool,
) -> Tuple[bool, Path]:
2022-02-10 16:34:55 +00:00
dummy_source = "ubuntu:20.04"
logging.info("Building docker image %s as %s", image.repo, dummy_source)
build_log = (
Path(TEMP_PATH)
/ f"build_and_push_log_{image.repo.replace('/', '_')}_{version_string}.log"
2022-02-10 16:34:55 +00:00
)
with open(build_log, "wb") as bl:
cmd = (
f"docker pull {dummy_source}; "
f"docker tag {dummy_source} {image.repo}:{version_string}; "
)
if push:
cmd += f"docker push {image.repo}:{version_string}"
logging.info("Docker command to run: %s", cmd)
with subprocess.Popen(cmd, shell=True, stderr=bl, stdout=bl) as proc:
retcode = proc.wait()
if retcode != 0:
return False, build_log
logging.info("Processing of %s successfully finished", image.repo)
return True, build_log
def build_and_push_one_image(
image: DockerImage,
version_string: str,
additional_cache: str,
push: bool,
child: bool,
) -> Tuple[bool, Path]:
2022-02-10 16:34:55 +00:00
if image.only_amd64 and platform.machine() not in ["amd64", "x86_64"]:
return build_and_push_dummy_image(image, version_string, push)
2021-12-21 16:32:04 +00:00
logging.info(
"Building docker image %s with version %s from path %s",
image.repo,
2021-12-21 16:32:04 +00:00
version_string,
image.full_path,
2021-12-21 16:32:04 +00:00
)
build_log = (
Path(TEMP_PATH)
/ f"build_and_push_log_{image.repo.replace('/', '_')}_{version_string}.log"
)
push_arg = ""
if push:
push_arg = "--push "
from_tag_arg = ""
if child:
from_tag_arg = f"--build-arg FROM_TAG={version_string} "
cache_from = (
f"--cache-from type=registry,ref={image.repo}:{version_string} "
f"--cache-from type=registry,ref={image.repo}:latest"
)
if additional_cache:
cache_from = (
f"{cache_from} "
f"--cache-from type=registry,ref={image.repo}:{additional_cache}"
)
2022-01-18 21:28:26 +00:00
with open(build_log, "wb") as bl:
cmd = (
2021-12-24 16:15:08 +00:00
"docker buildx build --builder default "
f"--label build-url={GITHUB_RUN_URL} "
f"{from_tag_arg}"
# A hack to invalidate cache, grep for it in docker/ dir
f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} "
f"--tag {image.repo}:{version_string} "
f"{cache_from} "
f"--cache-to type=inline,mode=max "
f"{push_arg}"
f"--progress plain {image.full_path}"
2021-12-21 16:32:04 +00:00
)
logging.info("Docker command to run: %s", cmd)
with subprocess.Popen(cmd, shell=True, stderr=bl, stdout=bl) as proc:
retcode = proc.wait()
2021-09-15 16:32:17 +00:00
if retcode != 0:
return False, build_log
2021-09-15 16:32:17 +00:00
logging.info("Processing of %s successfully finished", image.repo)
return True, build_log
2021-09-15 16:32:17 +00:00
2021-12-21 16:32:04 +00:00
def process_single_image(
image: DockerImage,
versions: List[str],
2022-11-10 15:57:01 +00:00
additional_cache: str,
push: bool,
child: bool,
) -> TestResults:
2021-12-21 16:32:04 +00:00
logging.info("Image will be pushed with versions %s", ", ".join(versions))
results = [] # type: TestResults
2021-09-15 16:32:17 +00:00
for ver in versions:
stopwatch = Stopwatch()
2021-09-15 16:32:17 +00:00
for i in range(5):
success, build_log = build_and_push_one_image(
image, ver, additional_cache, push, child
)
2021-09-15 16:32:17 +00:00
if success:
results.append(
TestResult(
image.repo + ":" + ver,
"OK",
stopwatch.duration_seconds,
[build_log],
)
)
2021-09-15 16:32:17 +00:00
break
2021-12-21 16:32:04 +00:00
logging.info(
"Got error will retry %s time and sleep for %s seconds", i, i * 5
)
2021-09-15 16:32:17 +00:00
time.sleep(i * 5)
else:
results.append(
TestResult(
image.repo + ":" + ver,
"FAIL",
stopwatch.duration_seconds,
[build_log],
)
)
2021-09-15 16:32:17 +00:00
logging.info("Processing finished")
image.built = True
return results
def process_image_with_parents(
image: DockerImage,
versions: List[str],
additional_cache: str,
push: bool,
child: bool = False,
) -> TestResults:
results = [] # type: TestResults
if image.built:
return results
if image.parent is not None:
results += process_image_with_parents(
image.parent, versions, additional_cache, push, False
)
child = True
results += process_single_image(image, versions, additional_cache, push, child)
return results
2021-09-15 16:32:17 +00:00
2021-12-21 16:32:04 +00:00
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Program to build changed or given docker images with all "
"dependant images. Example for local running: "
"python docker_images_check.py --no-push-images --no-reports "
"--image-path docker/packager/binary",
)
parser.add_argument(
"--suffix",
type=str,
help="suffix for all built images tags and resulting json file; the parameter "
"significantly changes the script behavior, e.g. changed_images.json is called "
"changed_images_{suffix}.json and contains list of all tags",
)
parser.add_argument(
"--repo",
type=str,
default="clickhouse",
help="docker hub repository prefix",
)
parser.add_argument(
"--all",
action="store_true",
help="rebuild all images",
)
parser.add_argument(
"--image-path",
type=str,
nargs="*",
help="list of image paths to build instead of using pr_info + diff URL, "
"e.g. 'docker/packager/binary'",
)
2022-02-21 11:44:37 +00:00
parser.add_argument("--reports", default=True, help=argparse.SUPPRESS)
parser.add_argument(
"--no-reports",
2022-02-21 11:44:37 +00:00
action="store_false",
dest="reports",
default=argparse.SUPPRESS,
help="don't push reports to S3 and github",
)
2022-02-21 11:44:37 +00:00
parser.add_argument("--push", default=True, help=argparse.SUPPRESS)
parser.add_argument(
"--no-push-images",
2022-02-21 11:44:37 +00:00
action="store_false",
dest="push",
default=argparse.SUPPRESS,
help="don't push images to docker hub",
)
return parser.parse_args()
2021-11-19 14:47:04 +00:00
def main():
logging.basicConfig(level=logging.INFO)
2021-11-19 14:47:04 +00:00
stopwatch = Stopwatch()
args = parse_args()
if args.suffix:
global NAME
NAME += f" {args.suffix}"
changed_json = os.path.join(TEMP_PATH, f"changed_images_{args.suffix}.json")
else:
changed_json = os.path.join(TEMP_PATH, "changed_images.json")
2022-02-21 11:44:37 +00:00
if args.push:
subprocess.check_output( # pylint: disable=unexpected-keyword-arg
"docker login --username 'robotclickhouse' --password-stdin",
input=get_parameter_from_ssm("dockerhub_robot_password"),
encoding="utf-8",
shell=True,
)
if os.path.exists(TEMP_PATH):
shutil.rmtree(TEMP_PATH)
os.makedirs(TEMP_PATH)
2021-09-16 10:52:04 +00:00
images_dict = get_images_dict(GITHUB_WORKSPACE, "docker/images.json")
2022-03-31 08:33:57 +00:00
pr_info = PRInfo()
if args.all:
pr_info.changed_files = set(images_dict.keys())
elif args.image_path:
pr_info.changed_files = set(i for i in args.image_path)
else:
try:
pr_info.fetch_changed_files()
except TypeError:
# If the event does not contain diff, nothing will be built
pass
2021-09-15 18:48:06 +00:00
changed_images = get_changed_docker_images(pr_info, images_dict)
2022-03-28 22:23:07 +00:00
if changed_images:
logging.info(
"Has changed images: %s", ", ".join([im.path for im in changed_images])
)
2021-09-15 16:32:17 +00:00
image_versions, result_version = gen_versions(pr_info, args.suffix)
2021-09-15 16:32:17 +00:00
2021-09-15 18:26:48 +00:00
result_images = {}
test_results = [] # type: TestResults
additional_cache = ""
if pr_info.release_pr or pr_info.merged_pr:
additional_cache = str(pr_info.release_pr or pr_info.merged_pr)
for image in changed_images:
# If we are in backport PR, then pr_info.release_pr is defined
# We use it as tag to reduce rebuilding time
test_results += process_image_with_parents(
image, image_versions, additional_cache, args.push
2021-12-21 16:32:04 +00:00
)
result_images[image.repo] = result_version
2021-09-15 16:32:17 +00:00
if changed_images:
description = "Updated " + ",".join([im.repo for im in changed_images])
2021-09-15 16:32:17 +00:00
else:
description = "Nothing to update"
if len(description) >= 140:
description = description[:136] + "..."
2022-01-18 21:28:26 +00:00
with open(changed_json, "w", encoding="utf-8") as images_file:
json.dump(result_images, images_file)
2022-08-11 13:01:32 +00:00
s3_helper = S3Helper()
2021-09-15 16:32:17 +00:00
status = "success"
if [r for r in test_results if r.status != "OK"]:
status = "failure"
2021-09-15 16:32:17 +00:00
2021-11-12 11:39:00 +00:00
url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [], NAME)
2021-09-15 16:32:17 +00:00
2022-01-18 21:28:26 +00:00
print(f"::notice ::Report url: {url}")
2022-02-21 11:44:37 +00:00
if not args.reports:
return
gh = Github(get_best_robot_token(), per_page=100)
post_commit_status(gh, pr_info.sha, NAME, description, status, url)
2021-12-21 16:32:04 +00:00
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
status,
stopwatch.duration_seconds,
stopwatch.start_time_str,
url,
NAME,
)
ch_helper = ClickHouseHelper()
2022-03-29 19:06:50 +00:00
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
if status == "failure":
2022-03-29 12:41:47 +00:00
sys.exit(1)
2022-03-29 16:23:18 +00:00
if __name__ == "__main__":
main()