From 9ca03d09cc3cc472315c4d60e46905ee6a0df279 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 8 Jun 2022 14:25:41 +0200 Subject: [PATCH] Add a way to invalidate docker images cache --- docker/docs/builder/Dockerfile | 9 +++++---- docker/docs/builder/run.sh | 2 -- tests/ci/docker_images_check.py | 14 ++++++++------ tests/ci/docker_test.py | 3 +++ 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/docker/docs/builder/Dockerfile b/docker/docs/builder/Dockerfile index a3b5bffdfe8..76bbea59e60 100644 --- a/docker/docs/builder/Dockerfile +++ b/docker/docs/builder/Dockerfile @@ -1,15 +1,16 @@ # rebuild in #36968 # docker build -t clickhouse/docs-builder . # nodejs 17 prefers ipv6 and is broken in our environment -FROM node:16.14.2-alpine3.15 +FROM node:16-alpine RUN apk add --no-cache git openssh bash -# TODO: clean before merge! -ARG DOCS_BRANCH=main +# At this point we want to really update /opt/clickhouse-docs +# despite the cached images +ARG CACHE_INVALIDATOR=0 RUN git clone https://github.com/ClickHouse/clickhouse-docs.git \ - --depth=1 --branch=${DOCS_BRANCH} /opt/clickhouse-docs + --depth=1 --branch=main /opt/clickhouse-docs WORKDIR /opt/clickhouse-docs diff --git a/docker/docs/builder/run.sh b/docker/docs/builder/run.sh index 2e13bfb55bb..a4f678b2f24 100755 --- a/docker/docs/builder/run.sh +++ b/docker/docs/builder/run.sh @@ -8,8 +8,6 @@ if [ "$GIT_DOCS_BRANCH" ] && ! [ "$GIT_DOCS_BRANCH" == "$GIT_BRANCH" ]; then git fetch origin --depth=1 -- "$GIT_DOCS_BRANCH:$GIT_DOCS_BRANCH" git checkout "$GIT_DOCS_BRANCH" else - # Untracked yarn.lock could cause pull to fail - git clean -fdx # Update docs repo git pull fi diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index c1d1c1df1f1..a0d0e124f6d 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -12,14 +12,14 @@ from typing import Dict, List, Optional, Set, Tuple, Union from github import Github -from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, GITHUB_RUN_URL -from s3_helper import S3Helper -from pr_info import PRInfo -from get_robot_token import get_best_robot_token, get_parameter_from_ssm -from upload_result_helper import upload_results -from commit_status_helper import post_commit_status from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse +from commit_status_helper import post_commit_status +from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, GITHUB_RUN_URL +from get_robot_token import get_best_robot_token, get_parameter_from_ssm +from pr_info import PRInfo +from s3_helper import S3Helper from stopwatch import Stopwatch +from upload_result_helper import upload_results NAME = "Push to Dockerhub (actions)" @@ -237,6 +237,8 @@ def build_and_push_one_image( "docker buildx build --builder default " f"--label build-url={GITHUB_RUN_URL} " f"{from_tag_arg}" + # A hack to invalidate cache, grep for it in docker/ dir + f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " f"--tag {image.repo}:{version_string} " f"--cache-from type=registry,ref={image.repo}:{version_string} " f"--cache-from type=registry,ref={image.repo}:latest " diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py index cabf12082e8..550d495939c 100644 --- a/tests/ci/docker_test.py +++ b/tests/ci/docker_test.py @@ -124,6 +124,7 @@ class TestDockerImageCheck(unittest.TestCase): self.assertIn( f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " "--build-arg FROM_TAG=version " + f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " "--tag name:version --cache-from type=registry,ref=name:version " "--cache-from type=registry,ref=name:latest " "--cache-to type=inline,mode=max --push --progress plain path", @@ -142,6 +143,7 @@ class TestDockerImageCheck(unittest.TestCase): self.assertIn( f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " "--build-arg FROM_TAG=version2 " + f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " "--tag name:version2 --cache-from type=registry,ref=name:version2 " "--cache-from type=registry,ref=name:latest " "--cache-to type=inline,mode=max --progress plain path", @@ -159,6 +161,7 @@ class TestDockerImageCheck(unittest.TestCase): mock_machine.assert_not_called() self.assertIn( f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " + f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " "--tag name:version2 --cache-from type=registry,ref=name:version2 " "--cache-from type=registry,ref=name:latest " "--cache-to type=inline,mode=max --progress plain path",