thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add a flag for images architecture

Browse Source
This commit is contained in:
Mikhail f. Shiryaev 2022-02-10 17:34:55 +01:00
parent 7ca684aa9c
commit 0f21981694
No known key found for this signature in database
GPG Key ID: 4B02ED204C7D93F4
4 changed files with 109 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docker/images.json
View File

@ -32,6 +32,7 @@
"dependent": []
},
"docker/test/pvs": {
"only_amd64": true,
"name": "clickhouse/pvs-test",
"dependent": []
},
@ -72,6 +73,7 @@
"dependent": []
},
"docker/test/integration/runner": {
"only_amd64": true,
"name": "clickhouse/integration-tests-runner",
"dependent": []
},
@ -124,6 +126,7 @@
"dependent": []
},
"docker/test/integration/kerberos_kdc": {
"only_amd64": true,
"name": "clickhouse/kerberos-kdc",
"dependent": []
},
@ -137,6 +140,7 @@
]
},
"docker/test/integration/kerberized_hadoop": {
"only_amd64": true,
"name": "clickhouse/kerberized-hadoop",
"dependent": []
},

52
tests/ci/docker_images_check.py
View File

@ -3,6 +3,7 @@ import argparse
import json
import logging
import os
import platform
import shutil
import subprocess
import time
@ -23,7 +24,7 @@ NAME = "Push to Dockerhub (actions)"
TEMP_PATH = os.path.join(RUNNER_TEMP, "docker_images_check")
ImagesDict = Dict[str, Dict[str, Union[str, List[str]]]]
ImagesDict = Dict[str, dict]
class DockerImage:
@ -31,18 +32,24 @@ class DockerImage:
self,
path: str,
repo: str,
only_amd64: bool,
parent: Optional["DockerImage"] = None,
gh_repo_path: str = GITHUB_WORKSPACE,
):
self.path = path
self.full_path = os.path.join(gh_repo_path, path)
self.repo = repo
self.only_amd64 = only_amd64
self.parent = parent
self.built = False
def __eq__(self, other) -> bool: # type: ignore
"""Is used to check if DockerImage is in a set or not"""
return self.path == other.path and self.repo == self.repo
return (
self.path == other.path
and self.repo == self.repo
and self.only_amd64 == other.only_amd64
)
def __lt__(self, other) -> bool:
if not isinstance(other, DockerImage):
@ -68,6 +75,7 @@ class DockerImage:
def get_images_dict(repo_path: str, image_file_path: str) -> ImagesDict:
"""Return images suppose to build on the current architecture host"""
images_dict = {}
path_to_images_file = os.path.join(repo_path, image_file_path)
if os.path.exists(path_to_images_file):
@ -103,6 +111,7 @@ def get_changed_docker_images(
for f in files_changed:
if f.startswith(dockerfile_dir):
name = image_description["name"]
only_amd64 = image_description.get("only_amd64", False)
logging.info(
"Found changed file '%s' which affects "
"docker image '%s' with path '%s'",
@ -110,7 +119,7 @@ def get_changed_docker_images(
name,
dockerfile_dir,
)
changed_images.append(DockerImage(dockerfile_dir, name))
changed_images.append(DockerImage(dockerfile_dir, name, only_amd64))
break
# The order is important: dependents should go later than bases, so that
@ -125,9 +134,9 @@ def get_changed_docker_images(
dependent,
image,
)
changed_images.append(
DockerImage(dependent, images_dict[dependent]["name"], image)
)
name = images_dict[dependent]["name"]
only_amd64 = images_dict[dependent].get("only_amd64", False)
changed_images.append(DockerImage(dependent, name, only_amd64, image))
index += 1
if index > 5 * len(images_dict):
# Sanity check to prevent infinite loop.
@ -168,12 +177,43 @@ def gen_versions(
return versions, result_version
def build_and_push_dummy_image(
image: DockerImage,
version_string: str,
push: bool,
) -> Tuple[bool, str]:
dummy_source = "ubuntu:20.04"
logging.info("Building docker image %s as %s", image.repo, dummy_source)
build_log = os.path.join(
TEMP_PATH, f"build_and_push_log_{image.repo.replace('/', '_')}_{version_string}"
)
with open(build_log, "wb") as bl:
cmd = (
f"docker pull {dummy_source}; "
f"docker tag {dummy_source} {image.repo}:{version_string}; "
)
if push:
cmd += f"docker push {image.repo}:{version_string}"
logging.info("Docker command to run: %s", cmd)
with subprocess.Popen(cmd, shell=True, stderr=bl, stdout=bl) as proc:
retcode = proc.wait()
if retcode != 0:
return False, build_log
logging.info("Processing of %s successfully finished", image.repo)
return True, build_log
def build_and_push_one_image(
image: DockerImage,
version_string: str,
push: bool,
child: bool,
) -> Tuple[bool, str]:
if image.only_amd64 and platform.machine() not in ["amd64", "x86_64"]:
return build_and_push_dummy_image(image, version_string, push)
logging.info(
"Building docker image %s with version %s from path %s",
image.repo,

69
tests/ci/docker_test.py
View File

@ -32,49 +32,60 @@ class TestDockerImageCheck(unittest.TestCase):
self.maxDiff = None
expected = sorted(
[
di.DockerImage("docker/test/base", "clickhouse/test-base"),
di.DockerImage("docker/docs/builder", "clickhouse/docs-builder"),
di.DockerImage("docker/test/base", "clickhouse/test-base", False),
di.DockerImage("docker/docs/builder", "clickhouse/docs-builder", True),
di.DockerImage(
"docker/test/stateless",
"clickhouse/stateless-test",
False,
"clickhouse/test-base",
),
di.DockerImage(
"docker/test/integration/base",
"clickhouse/integration-test",
False,
"clickhouse/test-base",
),
di.DockerImage(
"docker/test/fuzzer", "clickhouse/fuzzer", "clickhouse/test-base"
"docker/test/fuzzer",
"clickhouse/fuzzer",
False,
"clickhouse/test-base",
),
di.DockerImage(
"docker/test/keeper-jepsen",
"clickhouse/keeper-jepsen-test",
False,
"clickhouse/test-base",
),
di.DockerImage(
"docker/docs/check",
"clickhouse/docs-check",
False,
"clickhouse/docs-builder",
),
di.DockerImage(
"docker/docs/release",
"clickhouse/docs-release",
False,
"clickhouse/docs-builder",
),
di.DockerImage(
"docker/test/stateful",
"clickhouse/stateful-test",
False,
"clickhouse/stateless-test",
),
di.DockerImage(
"docker/test/unit",
"clickhouse/unit-test",
False,
"clickhouse/stateless-test",
),
di.DockerImage(
"docker/test/stress",
"clickhouse/stress-test",
False,
"clickhouse/stateful-test",
),
]
@ -96,13 +107,15 @@ class TestDockerImageCheck(unittest.TestCase):
@patch("builtins.open")
@patch("subprocess.Popen")
def test_build_and_push_one_image(self, mock_popen, mock_open):
@patch("platform.machine")
def test_build_and_push_one_image(self, mock_machine, mock_popen, mock_open):
mock_popen.return_value.__enter__.return_value.wait.return_value = 0
image = di.DockerImage("path", "name", gh_repo_path="")
image = di.DockerImage("path", "name", False, gh_repo_path="")
result, _ = di.build_and_push_one_image(image, "version", True, True)
mock_open.assert_called_once()
mock_popen.assert_called_once()
mock_machine.assert_not_called()
self.assertIn(
"docker buildx build --builder default --build-arg FROM_TAG=version "
"--build-arg BUILDKIT_INLINE_CACHE=1 --tag name:version --cache-from "
@ -110,11 +123,15 @@ class TestDockerImageCheck(unittest.TestCase):
mock_popen.call_args.args,
)
self.assertTrue(result)
mock_open.reset_mock()
mock_popen.reset_mock()
mock_machine.reset_mock()
mock_open.reset()
mock_popen.reset()
mock_popen.return_value.__enter__.return_value.wait.return_value = 0
result, _ = di.build_and_push_one_image(image, "version2", False, True)
mock_open.assert_called_once()
mock_popen.assert_called_once()
mock_machine.assert_not_called()
self.assertIn(
"docker buildx build --builder default --build-arg FROM_TAG=version2 "
"--build-arg BUILDKIT_INLINE_CACHE=1 --tag name:version2 --cache-from "
@ -123,8 +140,14 @@ class TestDockerImageCheck(unittest.TestCase):
)
self.assertTrue(result)
mock_open.reset_mock()
mock_popen.reset_mock()
mock_machine.reset_mock()
mock_popen.return_value.__enter__.return_value.wait.return_value = 1
result, _ = di.build_and_push_one_image(image, "version2", False, False)
mock_open.assert_called_once()
mock_popen.assert_called_once()
mock_machine.assert_not_called()
self.assertIn(
"docker buildx build --builder default "
"--build-arg BUILDKIT_INLINE_CACHE=1 --tag name:version2 --cache-from "
@ -133,13 +156,37 @@ class TestDockerImageCheck(unittest.TestCase):
)
self.assertFalse(result)
mock_open.reset_mock()
mock_popen.reset_mock()
mock_machine.reset_mock()
only_amd64_image = di.DockerImage("path", "name", True)
mock_popen.return_value.__enter__.return_value.wait.return_value = 0
result, _ = di.build_and_push_one_image(only_amd64_image, "version", True, True)
mock_open.assert_called_once()
mock_popen.assert_called_once()
mock_machine.assert_called_once()
self.assertIn(
"docker pull ubuntu:20.04; docker tag ubuntu:20.04 name:version; "
"docker push name:version",
mock_popen.call_args.args,
)
self.assertTrue(result)
result, _ = di.build_and_push_one_image(
only_amd64_image, "version", False, True
)
self.assertIn(
"docker pull ubuntu:20.04; docker tag ubuntu:20.04 name:version; ",
mock_popen.call_args.args,
)
@patch("docker_images_check.build_and_push_one_image")
def test_process_image_with_parents(self, mock_build):
mock_build.side_effect = lambda w, x, y, z: (True, f"{w.repo}_{x}.log")
im1 = di.DockerImage("path1", "repo1")
im2 = di.DockerImage("path2", "repo2", im1)
im3 = di.DockerImage("path3", "repo3", im2)
im4 = di.DockerImage("path4", "repo4", im1)
im1 = di.DockerImage("path1", "repo1", False)
im2 = di.DockerImage("path2", "repo2", False, im1)
im3 = di.DockerImage("path3", "repo3", False, im2)
im4 = di.DockerImage("path4", "repo4", False, im1)
# We use list to have determined order of image builgings
images = [im4, im1, im3, im2, im1]
results = [

1
tests/ci/tests/docker_images.json
View File

@ -150,6 +150,7 @@
},
"docker/docs/builder": {
"name": "clickhouse/docs-builder",
"only_amd64": true,
"dependent": [
"docker/docs/check",
"docker/docs/release"