Use test-util as source for base-test, fasttest and package builder

2024-09-19 16:20:50 +00:00 · 2022-07-29 14:31:53 +02:00 · 2022-07-29 14:31:53 +02:00 · eeaf08525f
commit eeaf08525f
parent 14135927fb
5 changed files with 84 additions and 168 deletions
--- a/docker/images.json
+++ b/docker/images.json
@ -29,6 +29,7 @@
    "docker/test/util": {
        "name": "clickhouse/test-util",
        "dependent": [
+            "docker/packager/binary",
            "docker/test/base",
            "docker/test/fasttest"
        ]
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@ -1,62 +1,7 @@
 # rebuild in #33610
 # docker build -t clickhouse/binary-builder .
-FROM ubuntu:20.04
-
-# ARG for quick switch to a given ubuntu mirror
-ARG apt_archive="http://archive.ubuntu.com"
-RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
-
-ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
-
-RUN apt-get update \
-    && apt-get install \
-        apt-transport-https \
-        apt-utils \
-        ca-certificates \
-        dnsutils \
-        gnupg \
-        iputils-ping \
-        lsb-release \
-        wget \
-        --yes --no-install-recommends --verbose-versions \
-    && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
-    && wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
-    && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
-    && apt-key add /tmp/llvm-snapshot.gpg.key \
-    && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
-    && echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
-        /etc/apt/sources.list \
-    && apt-get clean
-
-# initial packages
-RUN apt-get update \
-    && apt-get install \
-        bash \
-        build-essential \
-        ccache \
-        clang-${LLVM_VERSION} \
-        clang-tidy-${LLVM_VERSION} \
-        cmake \
-        curl \
-        fakeroot \
-        gdb \
-        git \
-        gperf \
-        lld-${LLVM_VERSION} \
-        llvm-${LLVM_VERSION} \
-        llvm-${LLVM_VERSION}-dev \
-        moreutils \
-        ninja-build \
-        pigz \
-        rename \
-        software-properties-common \
-        tzdata \
-        nasm \
-        --yes --no-install-recommends \
-    && apt-get clean
-
-# This symlink required by gcc to find lld compiler
-RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
+ARG FROM_TAG=latest
+FROM clickhouse/test-util:$FROM_TAG

 ENV CC=clang-${LLVM_VERSION}
 ENV CXX=clang++-${LLVM_VERSION}
--- a/docker/test/base/Dockerfile
+++ b/docker/test/base/Dockerfile
@ -3,59 +3,12 @@
 ARG FROM_TAG=latest
 FROM clickhouse/test-util:$FROM_TAG

-# ARG for quick switch to a given ubuntu mirror
-ARG apt_archive="http://archive.ubuntu.com"
-RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
-
-ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
-
-RUN apt-get update \
-    && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \
-        --yes --no-install-recommends --verbose-versions \
-    && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
-    && wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
-    && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
-    && apt-key add /tmp/llvm-snapshot.gpg.key \
-    && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
-    && echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
-        /etc/apt/sources.list
-
-# initial packages
 RUN apt-get update \
    && apt-get install \
-        bash \
-        fakeroot \
-        ccache \
-        curl \
-        software-properties-common \
-        --yes --no-install-recommends
-
-# Architecture of the image when BuildKit/buildx is used
-ARG TARGETARCH
-
-# Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able
-# to compress files using pigz (https://zlib.net/pigz/) instead of gzip.
-# Significantly increase deb packaging speed and compatible with old systems
-RUN arch=${TARGETARCH:-amd64} \
-  && curl -Lo /usr/bin/dpkg-deb https://github.com/ClickHouse-Extras/dpkg/releases/download/1.21.1-clickhouse/dpkg-deb-${arch}
-
-RUN apt-get update \
-    && apt-get install \
-        clang-${LLVM_VERSION} \
-        debhelper \
-        devscripts \
-        gdb  \
-        git \
-        gperf \
        lcov \
-        llvm-${LLVM_VERSION} \
-        moreutils \
+        netbase \
        perl \
-        pigz \
-        pkg-config \
-        tzdata \
        pv \
-        nasm \
        --yes --no-install-recommends

 # Sanitizer options for services (clickhouse-server)
--- a/docker/test/fasttest/Dockerfile
+++ b/docker/test/fasttest/Dockerfile
@ -3,83 +3,23 @@
 ARG FROM_TAG=latest
 FROM clickhouse/test-util:$FROM_TAG

-# ARG for quick switch to a given ubuntu mirror
-ARG apt_archive="http://archive.ubuntu.com"
-RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
-
-ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
-
-RUN apt-get update \
-    && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \
-        --yes --no-install-recommends --verbose-versions \
-    && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
-    && wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
-    && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
-    && apt-key add /tmp/llvm-snapshot.gpg.key \
-    && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
-    && echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
-        /etc/apt/sources.list
-
-# initial packages
 RUN apt-get update \
    && apt-get install \
-        bash \
-        fakeroot \
-        ccache \
-        curl \
-        software-properties-common \
-        --yes --no-install-recommends
-
-# Architecture of the image when BuildKit/buildx is used
-ARG TARGETARCH
-
-# Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able
-# to compress files using pigz (https://zlib.net/pigz/) instead of gzip.
-# Significantly increase deb packaging speed and compatible with old systems
-RUN arch=${TARGETARCH:-amd64} \
-  && curl -Lo /usr/bin/dpkg-deb https://github.com/ClickHouse-Extras/dpkg/releases/download/1.21.1-clickhouse/dpkg-deb-${arch}
-
-RUN apt-get update \
-    && apt-get install \
-        apt-transport-https \
-        bash \
        brotli \
-        build-essential \
-        ca-certificates \
-        ccache \
-        clang-${LLVM_VERSION} \
-        clang-tidy-${LLVM_VERSION} \
-        cmake \
-        curl \
        expect \
-        fakeroot \
-        gdb \
-        git \
-        gperf \
-        lld-${LLVM_VERSION} \
-        llvm-${LLVM_VERSION} \
+        file \
        lsof \
-        moreutils \
-        ninja-build \
        psmisc \
        python3 \
        python3-lxml \
        python3-pip \
        python3-requests \
        python3-termcolor \
-        rename \
-        software-properties-common \
-        tzdata \
        unixodbc \
-        file \
-        nasm \
       --yes --no-install-recommends

 RUN pip3 install numpy scipy pandas Jinja2

-# This symlink required by gcc to find lld compiler
-RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
-
 ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"

 RUN mkdir -p /tmp/clickhouse-odbc-tmp \
--- a/docker/test/util/Dockerfile
+++ b/docker/test/util/Dockerfile
@ -1,5 +1,82 @@
-# rebuild in #33610
 # docker build -t clickhouse/test-util .
-
 FROM ubuntu:20.04
+
+# ARG for quick switch to a given ubuntu mirror
+ARG apt_archive="http://archive.ubuntu.com"
+RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
+
+ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
+
+RUN apt-get update \
+    && apt-get install \
+        apt-transport-https \
+        apt-utils \
+        ca-certificates \
+        dnsutils \
+        gnupg \
+        iputils-ping \
+        lsb-release \
+        wget \
+        --yes --no-install-recommends --verbose-versions \
+    && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
+    && wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
+    && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
+    && apt-key add /tmp/llvm-snapshot.gpg.key \
+    && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
+    && echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
+        /etc/apt/sources.list \
+    && apt-get clean
+
+# initial packages
+RUN apt-get update \
+    && apt-get install \
+        bash \
+        bsdmainutils \
+        build-essential \
+        clang-${LLVM_VERSION} \
+        clang-tidy-${LLVM_VERSION} \
+        cmake \
+        curl \
+        fakeroot \
+        gdb \
+        git \
+        gperf \
+        lld-${LLVM_VERSION} \
+        llvm-${LLVM_VERSION} \
+        llvm-${LLVM_VERSION}-dev \
+        moreutils \
+        nasm \
+        ninja-build \
+        pigz \
+        rename \
+        software-properties-common \
+        tzdata \
+        --yes --no-install-recommends \
+    && apt-get clean
+
+# This symlink required by gcc to find lld compiler
+RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
+
+ARG CCACHE_VERSION=4.6.1
+RUN mkdir /tmp/ccache \
+    && cd /tmp/ccache \
+    && curl -L \
+        -O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz \
+        -O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz.asc \
+    && gpg --recv-keys --keyserver hkps://keyserver.ubuntu.com 5A939A71A46792CF57866A51996DDA075594ADB8 \
+    && gpg --verify ccache-4.6.1.tar.xz.asc \
+    && tar xf ccache-$CCACHE_VERSION.tar.xz \
+    && cd /tmp/ccache/ccache-$CCACHE_VERSION \
+    && cmake -DCMAKE_INSTALL_PREFIX=/usr \
+        -DCMAKE_BUILD_TYPE=None \
+        -DZSTD_FROM_INTERNET=ON \
+        -DREDIS_STORAGE_BACKEND=OFF \
+        -Wno-dev \
+        -B build \
+        -S . \
+    && make VERBOSE=1 -C build \
+    && make install -C build \
+    && cd / \
+    && rm -rf /tmp/ccache
+
 COPY process_functional_tests_result.py /