Merge pull request #39709 from ClickHouse/update-ccache

Update ccache to the latest available version
2024-11-22 15:42:02 +00:00 · 2022-08-04 00:21:44 +02:00 · 2022-08-04 00:21:44 +02:00 · 1842a3fc7a
commit 1842a3fc7a
parent 4354e3db96 8533769132
8 changed files with 92 additions and 175 deletions
--- a/docker/images.json
+++ b/docker/images.json
@ -29,6 +29,7 @@
    "docker/test/util": {
        "name": "clickhouse/test-util",
        "dependent": [
+            "docker/packager/binary",
            "docker/test/base",
            "docker/test/fasttest"
        ]
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@ -1,62 +1,7 @@
 # rebuild in #33610
 # docker build -t clickhouse/binary-builder .
-FROM ubuntu:20.04
-
-# ARG for quick switch to a given ubuntu mirror
-ARG apt_archive="http://archive.ubuntu.com"
-RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
-
-ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
-
-RUN apt-get update \
-    && apt-get install \
-        apt-transport-https \
-        apt-utils \
-        ca-certificates \
-        dnsutils \
-        gnupg \
-        iputils-ping \
-        lsb-release \
-        wget \
-        --yes --no-install-recommends --verbose-versions \
-    && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
-    && wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
-    && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
-    && apt-key add /tmp/llvm-snapshot.gpg.key \
-    && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
-    && echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
-        /etc/apt/sources.list \
-    && apt-get clean
-
-# initial packages
-RUN apt-get update \
-    && apt-get install \
-        bash \
-        build-essential \
-        ccache \
-        clang-${LLVM_VERSION} \
-        clang-tidy-${LLVM_VERSION} \
-        cmake \
-        curl \
-        fakeroot \
-        gdb \
-        git \
-        gperf \
-        lld-${LLVM_VERSION} \
-        llvm-${LLVM_VERSION} \
-        llvm-${LLVM_VERSION}-dev \
-        moreutils \
-        ninja-build \
-        pigz \
-        rename \
-        software-properties-common \
-        tzdata \
-        nasm \
-        --yes --no-install-recommends \
-    && apt-get clean
-
-# This symlink required by gcc to find lld compiler
-RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
+ARG FROM_TAG=latest
+FROM clickhouse/test-util:$FROM_TAG

 ENV CC=clang-${LLVM_VERSION}
 ENV CXX=clang++-${LLVM_VERSION}
--- a/docker/packager/binary/build.sh
+++ b/docker/packager/binary/build.sh
@ -3,7 +3,7 @@ set -x -e

 exec &> >(ts)

-cache_status () {
+ccache_status () {
    ccache --show-config ||:
    ccache --show-stats ||:
 }
@ -48,7 +48,7 @@ if [ -n "$MAKE_DEB" ]; then
 fi


-cache_status
+ccache_status
 # clear cache stats
 ccache --zero-stats ||:

@ -92,7 +92,7 @@ $SCAN_WRAPPER ninja $NINJA_FLAGS $BUILD_TARGET

 ls -la ./programs

-cache_status
+ccache_status

 if [ -n "$MAKE_DEB" ]; then
  # No quotes because I want it to expand to nothing if empty.
@ -178,7 +178,8 @@ then
    mv "coverity-scan.tgz" /output
 fi

-cache_status
+ccache_status
+ccache --evict-older-than 1d

 if [ "${CCACHE_DEBUG:-}" == "1" ]
 then
--- a/docker/packager/packager
+++ b/docker/packager/packager
@ -234,6 +234,7 @@ def parse_env_variables(

    if cache:
        result.append("CCACHE_DIR=/ccache")
+        result.append("CCACHE_COMPRESSLEVEL=5")
        result.append("CCACHE_BASEDIR=/build")
        result.append("CCACHE_NOHASHDIR=true")
        result.append("CCACHE_COMPILERCHECK=content")
@ -242,7 +243,6 @@ def parse_env_variables(
            # 15G is not enough for tidy build
            cache_maxsize = "25G"
        result.append(f"CCACHE_MAXSIZE={cache_maxsize}")
-        # result.append("CCACHE_UMASK=777")

    if distcc_hosts:
        hosts_with_params = [f"{host}/24,lzo" for host in distcc_hosts] + [
--- a/docker/test/base/Dockerfile
+++ b/docker/test/base/Dockerfile
@ -3,59 +3,12 @@
 ARG FROM_TAG=latest
 FROM clickhouse/test-util:$FROM_TAG

-# ARG for quick switch to a given ubuntu mirror
-ARG apt_archive="http://archive.ubuntu.com"
-RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
-
-ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
-
-RUN apt-get update \
-    && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \
-        --yes --no-install-recommends --verbose-versions \
-    && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
-    && wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
-    && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
-    && apt-key add /tmp/llvm-snapshot.gpg.key \
-    && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
-    && echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
-        /etc/apt/sources.list
-
-# initial packages
 RUN apt-get update \
    && apt-get install \
-        bash \
-        fakeroot \
-        ccache \
-        curl \
-        software-properties-common \
-        --yes --no-install-recommends
-
-# Architecture of the image when BuildKit/buildx is used
-ARG TARGETARCH
-
-# Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able
-# to compress files using pigz (https://zlib.net/pigz/) instead of gzip.
-# Significantly increase deb packaging speed and compatible with old systems
-RUN arch=${TARGETARCH:-amd64} \
-  && curl -Lo /usr/bin/dpkg-deb https://github.com/ClickHouse-Extras/dpkg/releases/download/1.21.1-clickhouse/dpkg-deb-${arch}
-
-RUN apt-get update \
-    && apt-get install \
-        clang-${LLVM_VERSION} \
-        debhelper \
-        devscripts \
-        gdb  \
-        git \
-        gperf \
        lcov \
-        llvm-${LLVM_VERSION} \
-        moreutils \
+        netbase \
        perl \
-        pigz \
-        pkg-config \
-        tzdata \
        pv \
-        nasm \
        --yes --no-install-recommends

 # Sanitizer options for services (clickhouse-server)
--- a/docker/test/fasttest/Dockerfile
+++ b/docker/test/fasttest/Dockerfile
@ -3,83 +3,23 @@
 ARG FROM_TAG=latest
 FROM clickhouse/test-util:$FROM_TAG

-# ARG for quick switch to a given ubuntu mirror
-ARG apt_archive="http://archive.ubuntu.com"
-RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
-
-ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
-
-RUN apt-get update \
-    && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \
-        --yes --no-install-recommends --verbose-versions \
-    && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
-    && wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
-    && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
-    && apt-key add /tmp/llvm-snapshot.gpg.key \
-    && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
-    && echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
-        /etc/apt/sources.list
-
-# initial packages
 RUN apt-get update \
    && apt-get install \
-        bash \
-        fakeroot \
-        ccache \
-        curl \
-        software-properties-common \
-        --yes --no-install-recommends
-
-# Architecture of the image when BuildKit/buildx is used
-ARG TARGETARCH
-
-# Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able
-# to compress files using pigz (https://zlib.net/pigz/) instead of gzip.
-# Significantly increase deb packaging speed and compatible with old systems
-RUN arch=${TARGETARCH:-amd64} \
-  && curl -Lo /usr/bin/dpkg-deb https://github.com/ClickHouse-Extras/dpkg/releases/download/1.21.1-clickhouse/dpkg-deb-${arch}
-
-RUN apt-get update \
-    && apt-get install \
-        apt-transport-https \
-        bash \
        brotli \
-        build-essential \
-        ca-certificates \
-        ccache \
-        clang-${LLVM_VERSION} \
-        clang-tidy-${LLVM_VERSION} \
-        cmake \
-        curl \
        expect \
-        fakeroot \
-        gdb \
-        git \
-        gperf \
-        lld-${LLVM_VERSION} \
-        llvm-${LLVM_VERSION} \
+        file \
        lsof \
-        moreutils \
-        ninja-build \
        psmisc \
        python3 \
        python3-lxml \
        python3-pip \
        python3-requests \
        python3-termcolor \
-        rename \
-        software-properties-common \
-        tzdata \
        unixodbc \
-        file \
-        nasm \
       --yes --no-install-recommends

 RUN pip3 install numpy scipy pandas Jinja2

-# This symlink required by gcc to find lld compiler
-RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
-
 ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"

 RUN mkdir -p /tmp/clickhouse-odbc-tmp \
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@ -160,9 +160,8 @@ function run_cmake
        "-DENABLE_REPLXX=1"
    )

-    # TODO remove this? we don't use ccache anyway. An option would be to download it
-    # from S3 simultaneously with cloning.
    export CCACHE_DIR="$FASTTEST_WORKSPACE/ccache"
+    export CCACHE_COMPRESSLEVEL=5
    export CCACHE_BASEDIR="$FASTTEST_SOURCE"
    export CCACHE_NOHASHDIR=true
    export CCACHE_COMPILERCHECK=content
@ -191,6 +190,7 @@ function build
            gzip "$FASTTEST_OUTPUT/clickhouse-stripped"
        fi
        ccache --show-stats ||:
+        ccache --evict-older-than 1d ||:
    )
 }

--- a/docker/test/util/Dockerfile
+++ b/docker/test/util/Dockerfile
@ -1,5 +1,82 @@
-# rebuild in #33610
 # docker build -t clickhouse/test-util .
-
 FROM ubuntu:20.04
+
+# ARG for quick switch to a given ubuntu mirror
+ARG apt_archive="http://archive.ubuntu.com"
+RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
+
+ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
+
+RUN apt-get update \
+    && apt-get install \
+        apt-transport-https \
+        apt-utils \
+        ca-certificates \
+        dnsutils \
+        gnupg \
+        iputils-ping \
+        lsb-release \
+        wget \
+        --yes --no-install-recommends --verbose-versions \
+    && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
+    && wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
+    && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
+    && apt-key add /tmp/llvm-snapshot.gpg.key \
+    && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
+    && echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
+        /etc/apt/sources.list \
+    && apt-get clean
+
+# initial packages
+RUN apt-get update \
+    && apt-get install \
+        bash \
+        bsdmainutils \
+        build-essential \
+        clang-${LLVM_VERSION} \
+        clang-tidy-${LLVM_VERSION} \
+        cmake \
+        curl \
+        fakeroot \
+        gdb \
+        git \
+        gperf \
+        lld-${LLVM_VERSION} \
+        llvm-${LLVM_VERSION} \
+        llvm-${LLVM_VERSION}-dev \
+        moreutils \
+        nasm \
+        ninja-build \
+        pigz \
+        rename \
+        software-properties-common \
+        tzdata \
+        --yes --no-install-recommends \
+    && apt-get clean
+
+# This symlink required by gcc to find lld compiler
+RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
+
+ARG CCACHE_VERSION=4.6.1
+RUN mkdir /tmp/ccache \
+    && cd /tmp/ccache \
+    && curl -L \
+        -O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz \
+        -O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz.asc \
+    && gpg --recv-keys --keyserver hkps://keyserver.ubuntu.com 5A939A71A46792CF57866A51996DDA075594ADB8 \
+    && gpg --verify ccache-4.6.1.tar.xz.asc \
+    && tar xf ccache-$CCACHE_VERSION.tar.xz \
+    && cd /tmp/ccache/ccache-$CCACHE_VERSION \
+    && cmake -DCMAKE_INSTALL_PREFIX=/usr \
+        -DCMAKE_BUILD_TYPE=None \
+        -DZSTD_FROM_INTERNET=ON \
+        -DREDIS_STORAGE_BACKEND=OFF \
+        -Wno-dev \
+        -B build \
+        -S . \
+    && make VERBOSE=1 -C build \
+    && make install -C build \
+    && cd / \
+    && rm -rf /tmp/ccache
+
 COPY process_functional_tests_result.py /