thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #46246 from ClickHouse/optimize-binary-builder

Browse Source 
Optimize docker binary-builder
This commit is contained in:
robot-ch-test-poll3 2023-02-14 10:06:15 +01:00 committed by GitHub
commit f654174362
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 120 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
docker/packager/binary/Dockerfile
View File

@ -1,7 +1,47 @@
# rebuild in #33610
# docker build -t clickhouse/binary-builder .
ARG FROM_TAG=latest
FROM clickhouse/test-util:latest AS cctools
# The cctools are built always from the clickhouse/test-util:latest and cached inline
# Theoretically, it should improve rebuild speed significantly
ENV CC=clang-${LLVM_VERSION}
ENV CXX=clang++-${LLVM_VERSION}
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# DO NOT PUT ANYTHING BEFORE THREE NEXT `RUN` DIRECTIVES
# THE MOST HEAVY OPERATION MUST BE THE FIRST IN THE CACHE
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# libtapi is required to support .tbh format from recent MacOS SDKs
RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \
&& cd apple-libtapi \
&& INSTALLPREFIX=/cctools ./build.sh \
&& ./install.sh \
&& cd .. \
&& rm -rf apple-libtapi
# Build and install tools for cross-linking to Darwin (x86-64)
RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \
&& cd cctools-port/cctools \
&& ./configure --prefix=/cctools --with-libtapi=/cctools \
--target=x86_64-apple-darwin \
&& make install -j$(nproc) \
&& cd ../.. \
&& rm -rf cctools-port
# Build and install tools for cross-linking to Darwin (aarch64)
RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \
&& cd cctools-port/cctools \
&& ./configure --prefix=/cctools --with-libtapi=/cctools \
--target=aarch64-apple-darwin \
&& make install -j$(nproc) \
&& cd ../.. \
&& rm -rf cctools-port
# !!!!!!!!!!!
# END COMPILE
# !!!!!!!!!!!
FROM clickhouse/test-util:$FROM_TAG
ENV CC=clang-${LLVM_VERSION}
ENV CXX=clang++-${LLVM_VERSION}
# Rust toolchain and libraries
ENV RUSTUP_HOME=/rust/rustup
@ -16,80 +56,53 @@ RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \
rustup target add aarch64-apple-darwin && \
rustup target add powerpc64le-unknown-linux-gnu
RUN apt-get update && \
apt-get install --yes \
gcc-aarch64-linux-gnu \
# NOTE: Seems like gcc-11 is too new for ubuntu20 repository
# A cross-linker for RISC-V 64 (we need it, because LLVM's LLD does not work):
RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
&& apt-get update \
&& apt-get install --yes \
binutils-riscv64-linux-gnu \
build-essential \
g++-11 \
gcc-11 \
gcc-aarch64-linux-gnu \
libc6 \
libc6-dev \
libc6-dev-arm64-cross \
yasm \
zstd && \
apt-get clean
ENV CC=clang-${LLVM_VERSION}
ENV CXX=clang++-${LLVM_VERSION}
# libtapi is required to support .tbh format from recent MacOS SDKs
RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \
&& cd apple-libtapi \
&& INSTALLPREFIX=/cctools ./build.sh \
&& ./install.sh \
&& cd .. \
&& rm -rf apple-libtapi
# Build and install tools for cross-linking to Darwin (x86-64)
RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \
&& cd cctools-port/cctools \
&& ./configure --prefix=/cctools --with-libtapi=/cctools \
--target=x86_64-apple-darwin \
&& make install \
&& cd ../.. \
&& rm -rf cctools-port
# Build and install tools for cross-linking to Darwin (aarch64)
RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \
&& cd cctools-port/cctools \
&& ./configure --prefix=/cctools --with-libtapi=/cctools \
--target=aarch64-apple-darwin \
&& make install \
&& cd ../.. \
&& rm -rf cctools-port
zstd \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists
# Download toolchain and SDK for Darwin
RUN wget -nv https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz
# NOTE: Seems like gcc-11 is too new for ubuntu20 repository
RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
&& apt-get update \
&& apt-get install gcc-11 g++-11 --yes \
&& apt-get clean
# A cross-linker for RISC-V 64 (we need it, because LLVM's LLD does not work):
RUN apt-get install binutils-riscv64-linux-gnu
# Architecture of the image when BuildKit/buildx is used
ARG TARGETARCH
ARG NFPM_VERSION=2.20.0
RUN arch=${TARGETARCH:-amd64} \
&& curl -Lo /tmp/nfpm.deb "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${arch}.deb" \
&& dpkg -i /tmp/nfpm.deb \
&& rm /tmp/nfpm.deb
&& curl -Lo /tmp/nfpm.deb "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${arch}.deb" \
&& dpkg -i /tmp/nfpm.deb \
&& rm /tmp/nfpm.deb
ARG GO_VERSION=1.18.3
ARG GO_VERSION=1.19.5
# We need go for clickhouse-diagnostics
RUN arch=${TARGETARCH:-amd64} \
&& curl -Lo /tmp/go.tgz "https://go.dev/dl/go${GO_VERSION}.linux-${arch}.tar.gz" \
&& tar -xzf /tmp/go.tgz -C /usr/local/ \
&& rm /tmp/go.tgz
&& curl -Lo /tmp/go.tgz "https://go.dev/dl/go${GO_VERSION}.linux-${arch}.tar.gz" \
&& tar -xzf /tmp/go.tgz -C /usr/local/ \
&& rm /tmp/go.tgz
ENV PATH="$PATH:/usr/local/go/bin"
ENV GOPATH=/workdir/go
ENV GOCACHE=/workdir/
RUN curl https://raw.githubusercontent.com/matus-chochlik/ctcache/7fd516e91c17779cbc6fc18bd119313d9532dd90/clang-tidy-cache -Lo /usr/bin/clang-tidy-cache \
&& chmod +x /usr/bin/clang-tidy-cache
ARG CLANG_TIDY_SHA1=03644275e794b0587849bfc2ec6123d5ae0bdb1c
RUN curl -Lo /usr/bin/clang-tidy-cache \
"https://raw.githubusercontent.com/matus-chochlik/ctcache/$CLANG_TIDY_SHA1/clang-tidy-cache" \
&& chmod +x /usr/bin/clang-tidy-cache
COPY --from=cctools /cctools /cctools
RUN mkdir /workdir && chmod 777 /workdir
WORKDIR /workdir

83
tests/ci/docker_images_check.py
View File

@ -21,6 +21,7 @@ from pr_info import PRInfo
from report import TestResults, TestResult
from s3_helper import S3Helper
from stopwatch import Stopwatch
from tee_popen import TeePopen
from upload_result_helper import upload_results
NAME = "Push to Dockerhub"
@ -191,20 +192,19 @@ def build_and_push_dummy_image(
Path(TEMP_PATH)
/ f"build_and_push_log_{image.repo.replace('/', '_')}_{version_string}.log"
)
with open(build_log, "wb") as bl:
cmd = (
f"docker pull {dummy_source}; "
f"docker tag {dummy_source} {image.repo}:{version_string}; "
)
if push:
cmd += f"docker push {image.repo}:{version_string}"
cmd = (
f"docker pull {dummy_source}; "
f"docker tag {dummy_source} {image.repo}:{version_string}; "
)
if push:
cmd += f"docker push {image.repo}:{version_string}"
logging.info("Docker command to run: %s", cmd)
with subprocess.Popen(cmd, shell=True, stderr=bl, stdout=bl) as proc:
retcode = proc.wait()
logging.info("Docker command to run: %s", cmd)
with TeePopen(cmd, build_log) as proc:
retcode = proc.wait()
if retcode != 0:
return False, build_log
if retcode != 0:
return False, build_log
logging.info("Processing of %s successfully finished", image.repo)
return True, build_log
@ -213,7 +213,7 @@ def build_and_push_dummy_image(
def build_and_push_one_image(
image: DockerImage,
version_string: str,
additional_cache: str,
additional_cache: List[str],
push: bool,
child: bool,
) -> Tuple[bool, Path]:
@ -241,31 +241,28 @@ def build_and_push_one_image(
f"--cache-from type=registry,ref={image.repo}:{version_string} "
f"--cache-from type=registry,ref={image.repo}:latest"
)
if additional_cache:
cache_from = (
f"{cache_from} "
f"--cache-from type=registry,ref={image.repo}:{additional_cache}"
)
for tag in additional_cache:
assert tag
cache_from = f"{cache_from} --cache-from type=registry,ref={image.repo}:{tag}"
with open(build_log, "wb") as bl:
cmd = (
"docker buildx build --builder default "
f"--label build-url={GITHUB_RUN_URL} "
f"{from_tag_arg}"
# A hack to invalidate cache, grep for it in docker/ dir
f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} "
f"--tag {image.repo}:{version_string} "
f"{cache_from} "
f"--cache-to type=inline,mode=max "
f"{push_arg}"
f"--progress plain {image.full_path}"
)
logging.info("Docker command to run: %s", cmd)
with subprocess.Popen(cmd, shell=True, stderr=bl, stdout=bl) as proc:
retcode = proc.wait()
cmd = (
"docker buildx build --builder default "
f"--label build-url={GITHUB_RUN_URL} "
f"{from_tag_arg}"
# A hack to invalidate cache, grep for it in docker/ dir
f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} "
f"--tag {image.repo}:{version_string} "
f"{cache_from} "
f"--cache-to type=inline,mode=max "
f"{push_arg}"
f"--progress plain {image.full_path}"
)
logging.info("Docker command to run: %s", cmd)
with TeePopen(cmd, build_log) as proc:
retcode = proc.wait()
if retcode != 0:
return False, build_log
if retcode != 0:
return False, build_log
logging.info("Processing of %s successfully finished", image.repo)
return True, build_log
@ -274,7 +271,7 @@ def build_and_push_one_image(
def process_single_image(
image: DockerImage,
versions: List[str],
additional_cache: str,
additional_cache: List[str],
push: bool,
child: bool,
) -> TestResults:
@ -318,7 +315,7 @@ def process_single_image(
def process_image_with_parents(
image: DockerImage,
versions: List[str],
additional_cache: str,
additional_cache: List[str],
push: bool,
child: bool = False,
) -> TestResults:
@ -438,9 +435,13 @@ def main():
result_images = {}
test_results = [] # type: TestResults
additional_cache = ""
if pr_info.release_pr or pr_info.merged_pr:
additional_cache = str(pr_info.release_pr or pr_info.merged_pr)
additional_cache = [] # type: List[str]
if pr_info.release_pr:
logging.info("Use %s as additional cache tag", pr_info.release_pr)
additional_cache.append(str(pr_info.release_pr))
if pr_info.merged_pr:
logging.info("Use %s as additional cache tag", pr_info.merged_pr)
additional_cache.append(str(pr_info.merged_pr))
for image in changed_images:
# If we are in backport PR, then pr_info.release_pr is defined

31
tests/ci/docker_test.py
View File

@ -117,15 +117,13 @@ class TestDockerImageCheck(unittest.TestCase):
self.assertEqual(versions, ["1", "1-HEAD"])
self.assertEqual(result_version, "1-HEAD")
@patch("builtins.open")
@patch("subprocess.Popen")
@patch("docker_images_check.TeePopen")
@patch("platform.machine")
def test_build_and_push_one_image(self, mock_machine, mock_popen, mock_open):
def test_build_and_push_one_image(self, mock_machine, mock_popen):
mock_popen.return_value.__enter__.return_value.wait.return_value = 0
image = di.DockerImage("path", "name", False, gh_repo_path="")
result, _ = di.build_and_push_one_image(image, "version", "", True, True)
mock_open.assert_called_once()
result, _ = di.build_and_push_one_image(image, "version", [], True, True)
mock_popen.assert_called_once()
mock_machine.assert_not_called()
self.assertIn(
@ -138,13 +136,11 @@ class TestDockerImageCheck(unittest.TestCase):
mock_popen.call_args.args,
)
self.assertTrue(result)
mock_open.reset_mock()
mock_popen.reset_mock()
mock_machine.reset_mock()
mock_popen.return_value.__enter__.return_value.wait.return_value = 0
result, _ = di.build_and_push_one_image(image, "version2", "", False, True)
mock_open.assert_called_once()
result, _ = di.build_and_push_one_image(image, "version2", [], False, True)
mock_popen.assert_called_once()
mock_machine.assert_not_called()
self.assertIn(
@ -158,12 +154,10 @@ class TestDockerImageCheck(unittest.TestCase):
)
self.assertTrue(result)
mock_open.reset_mock()
mock_popen.reset_mock()
mock_machine.reset_mock()
mock_popen.return_value.__enter__.return_value.wait.return_value = 1
result, _ = di.build_and_push_one_image(image, "version2", "", False, False)
mock_open.assert_called_once()
result, _ = di.build_and_push_one_image(image, "version2", [], False, False)
mock_popen.assert_called_once()
mock_machine.assert_not_called()
self.assertIn(
@ -176,14 +170,12 @@ class TestDockerImageCheck(unittest.TestCase):
)
self.assertFalse(result)
mock_open.reset_mock()
mock_popen.reset_mock()
mock_machine.reset_mock()
mock_popen.return_value.__enter__.return_value.wait.return_value = 1
result, _ = di.build_and_push_one_image(
image, "version2", "cached-version", False, False
image, "version2", ["cached-version", "another-cached"], False, False
)
mock_open.assert_called_once()
mock_popen.assert_called_once()
mock_machine.assert_not_called()
self.assertIn(
@ -192,21 +184,20 @@ class TestDockerImageCheck(unittest.TestCase):
"--tag name:version2 --cache-from type=registry,ref=name:version2 "
"--cache-from type=registry,ref=name:latest "
"--cache-from type=registry,ref=name:cached-version "
"--cache-from type=registry,ref=name:another-cached "
"--cache-to type=inline,mode=max --progress plain path",
mock_popen.call_args.args,
)
self.assertFalse(result)
mock_open.reset_mock()
mock_popen.reset_mock()
mock_machine.reset_mock()
only_amd64_image = di.DockerImage("path", "name", True)
mock_popen.return_value.__enter__.return_value.wait.return_value = 0
result, _ = di.build_and_push_one_image(
only_amd64_image, "version", "", True, True
only_amd64_image, "version", [], True, True
)
mock_open.assert_called_once()
mock_popen.assert_called_once()
mock_machine.assert_called_once()
self.assertIn(
@ -216,12 +207,14 @@ class TestDockerImageCheck(unittest.TestCase):
)
self.assertTrue(result)
result, _ = di.build_and_push_one_image(
only_amd64_image, "version", "", False, True
only_amd64_image, "version", [], False, True
)
self.assertIn(
"docker pull ubuntu:20.04; docker tag ubuntu:20.04 name:version; ",
mock_popen.call_args.args,
)
with self.assertRaises(AssertionError):
result, _ = di.build_and_push_one_image(image, "version", [""], False, True)
@patch("docker_images_check.build_and_push_one_image")
def test_process_image_with_parents(self, mock_build):
@ -233,7 +226,7 @@ class TestDockerImageCheck(unittest.TestCase):
# We use list to have determined order of image builgings
images = [im4, im1, im3, im2, im1]
test_results = [
di.process_image_with_parents(im, ["v1", "v2", "latest"], "", True)
di.process_image_with_parents(im, ["v1", "v2", "latest"], [], True)
for im in images
]
# The time is random, so we check it's not None and greater than 0,