mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-12-16 03:12:43 +00:00
0b258dda4e
From now on cargo will not download anything from the internet during builds. This step had been moved for docker image builds (via cargo vendor). And now cargo inside docker.io/clickhouse/binary-builder will not use any crates from the internet, so we don't need to add --offline for cargo commands in cmake (corrosion_import_crate()). Also the docker build command had been adjusted to allow following symlinks inside build context, by using tar, this is required for Rust packages. Note, that to make proper Cargo.lock that could be vendored I did the following: - per-project locks had been removed (since there is no automatic way to sync the workspace Cargo.lock with per-project Cargo.lock, since cargo update/generate-lockfile will use only per-project Cargo.toml files apparently, -Z minimal-versions does not helps either) - and to generate Cargo.lock with less changes I've pinned version in the Cargo.toml strictly, i.e. not 'foo = "0.1"' but 'foo = "=0.1"' then the Cargo.lock for workspace had been generated and afterwards I've reverted this part. Plus I have to update the dependencies afterwards, since otherwise there are conflicts with dependencies for std library. Non trivial. Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
142 lines
5.3 KiB
Docker
142 lines
5.3 KiB
Docker
# docker build -t clickhouse/binary-builder .
|
|
ARG FROM_TAG=latest
|
|
FROM clickhouse/test-util:latest AS cctools
|
|
# The cctools are built always from the clickhouse/test-util:latest and cached inline
|
|
# Theoretically, it should improve rebuild speed significantly
|
|
ENV CC=clang-${LLVM_VERSION}
|
|
ENV CXX=clang++-${LLVM_VERSION}
|
|
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
# DO NOT PUT ANYTHING BEFORE THREE NEXT `RUN` DIRECTIVES
|
|
# THE MOST HEAVY OPERATION MUST BE THE FIRST IN THE CACHE
|
|
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
# libtapi is required to support .tbh format from recent MacOS SDKs
|
|
RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \
|
|
&& cd apple-libtapi \
|
|
&& INSTALLPREFIX=/cctools ./build.sh \
|
|
&& ./install.sh \
|
|
&& cd .. \
|
|
&& rm -rf apple-libtapi
|
|
|
|
# Build and install tools for cross-linking to Darwin (x86-64)
|
|
RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \
|
|
&& cd cctools-port/cctools \
|
|
&& ./configure --prefix=/cctools --with-libtapi=/cctools \
|
|
--target=x86_64-apple-darwin \
|
|
&& make install -j$(nproc) \
|
|
&& cd ../.. \
|
|
&& rm -rf cctools-port
|
|
|
|
# Build and install tools for cross-linking to Darwin (aarch64)
|
|
RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \
|
|
&& cd cctools-port/cctools \
|
|
&& ./configure --prefix=/cctools --with-libtapi=/cctools \
|
|
--target=aarch64-apple-darwin \
|
|
&& make install -j$(nproc) \
|
|
&& cd ../.. \
|
|
&& rm -rf cctools-port
|
|
|
|
# !!!!!!!!!!!
|
|
# END COMPILE
|
|
# !!!!!!!!!!!
|
|
|
|
FROM clickhouse/test-util:$FROM_TAG
|
|
ENV CC=clang-${LLVM_VERSION}
|
|
ENV CXX=clang++-${LLVM_VERSION}
|
|
|
|
# Rust toolchain and libraries
|
|
ENV RUSTUP_HOME=/rust/rustup
|
|
ENV CARGO_HOME=/rust/cargo
|
|
ENV PATH="/rust/cargo/bin:${PATH}"
|
|
RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \
|
|
chmod 777 -R /rust && \
|
|
rustup toolchain install nightly-2023-07-04 && \
|
|
rustup default nightly-2023-07-04 && \
|
|
rustup component add rust-src && \
|
|
rustup target add aarch64-unknown-linux-gnu && \
|
|
rustup target add x86_64-apple-darwin && \
|
|
rustup target add x86_64-unknown-freebsd && \
|
|
rustup target add aarch64-apple-darwin && \
|
|
rustup target add powerpc64le-unknown-linux-gnu
|
|
|
|
# Create vendor cache for cargo.
|
|
#
|
|
# Note, that the config.toml for the root is used, you will not be able to
|
|
# install any other crates, except those which had been vendored (since if
|
|
# there is "replace-with" for some source, then cargo will not look to other
|
|
# remotes except this).
|
|
#
|
|
# Notes for the command itself:
|
|
# - --chown is required to preserve the rights
|
|
# - unstable-options for -C
|
|
# - chmod is required to fix the permissions, since builds are running from a different user
|
|
# - copy of the Cargo.lock is required for proper dependencies versions
|
|
# - cargo vendor --sync is requried to overcome [1] bug.
|
|
#
|
|
# [1]: https://github.com/rust-lang/wg-cargo-std-aware/issues/23
|
|
COPY --chown=root:root /rust /rust/packages
|
|
RUN cargo -Z unstable-options -C /rust/packages vendor > $CARGO_HOME/config.toml && \
|
|
cp "$(rustc --print=sysroot)"/lib/rustlib/src/rust/Cargo.lock "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/ && \
|
|
cargo -Z unstable-options -C /rust/packages vendor --sync "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/Cargo.toml && \
|
|
rm "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/Cargo.lock && \
|
|
sed -i "s#\"vendor\"#\"/rust/vendor\"#" $CARGO_HOME/config.toml && \
|
|
cat $CARGO_HOME/config.toml && \
|
|
mv /rust/packages/vendor /rust/vendor && \
|
|
chmod -R o=r+X /rust/vendor && \
|
|
ls -R -l /rust/packages && \
|
|
rm -r /rust/packages
|
|
|
|
# NOTE: Seems like gcc-11 is too new for ubuntu20 repository
|
|
# A cross-linker for RISC-V 64 (we need it, because LLVM's LLD does not work):
|
|
RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
|
|
&& apt-get update \
|
|
&& apt-get install --yes \
|
|
binutils-riscv64-linux-gnu \
|
|
build-essential \
|
|
g++-11 \
|
|
gcc-11 \
|
|
gcc-aarch64-linux-gnu \
|
|
libc6 \
|
|
libc6-dev \
|
|
libc6-dev-arm64-cross \
|
|
python3-boto3 \
|
|
yasm \
|
|
zstd \
|
|
&& apt-get clean \
|
|
&& rm -rf /var/lib/apt/lists
|
|
|
|
# Download toolchain and SDK for Darwin
|
|
RUN curl -sL -O https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz
|
|
|
|
# Architecture of the image when BuildKit/buildx is used
|
|
ARG TARGETARCH
|
|
ARG NFPM_VERSION=2.20.0
|
|
|
|
RUN arch=${TARGETARCH:-amd64} \
|
|
&& curl -Lo /tmp/nfpm.deb "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${arch}.deb" \
|
|
&& dpkg -i /tmp/nfpm.deb \
|
|
&& rm /tmp/nfpm.deb
|
|
|
|
ARG GO_VERSION=1.19.10
|
|
# We need go for clickhouse-diagnostics
|
|
RUN arch=${TARGETARCH:-amd64} \
|
|
&& curl -Lo /tmp/go.tgz "https://go.dev/dl/go${GO_VERSION}.linux-${arch}.tar.gz" \
|
|
&& tar -xzf /tmp/go.tgz -C /usr/local/ \
|
|
&& rm /tmp/go.tgz
|
|
|
|
ENV PATH="$PATH:/usr/local/go/bin"
|
|
ENV GOPATH=/workdir/go
|
|
ENV GOCACHE=/workdir/
|
|
|
|
ARG CLANG_TIDY_SHA1=c191254ea00d47ade11d7170ef82fe038c213774
|
|
RUN curl -Lo /usr/bin/clang-tidy-cache \
|
|
"https://raw.githubusercontent.com/matus-chochlik/ctcache/$CLANG_TIDY_SHA1/clang-tidy-cache" \
|
|
&& chmod +x /usr/bin/clang-tidy-cache
|
|
|
|
COPY --from=cctools /cctools /cctools
|
|
|
|
RUN mkdir /workdir && chmod 777 /workdir
|
|
WORKDIR /workdir
|
|
|
|
COPY build.sh /
|
|
CMD ["bash", "-c", "/build.sh 2>&1"]
|