ClickHouse/docker/packager/binary/build.sh

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

198 lines
7.1 KiB
Bash
Raw Normal View History

2019-04-25 12:29:28 +00:00
#!/usr/bin/env bash
set -x -e
2019-04-25 12:29:28 +00:00
exec &> >(ts)
2019-04-25 12:29:28 +00:00
ccache_status () {
2022-02-08 18:12:04 +00:00
ccache --show-config ||:
ccache --show-stats ||:
}
[ -O /build ] || git config --global --add safe.directory /build
2022-04-13 19:33:34 +00:00
mkdir -p /build/cmake/toolchain/darwin-x86_64
tar xJf /MacOSX11.0.sdk.tar.xz -C /build/cmake/toolchain/darwin-x86_64 --strip-components=1
ln -sf darwin-x86_64 /build/cmake/toolchain/darwin-aarch64
2021-04-01 12:12:30 +00:00
# Uncomment to debug ccache. Don't put ccache log in /output right away, or it
# will be confusingly packed into the "performance" package.
2021-04-07 20:38:48 +00:00
# export CCACHE_LOGFILE=/build/ccache.log
# export CCACHE_DEBUG=1
2021-03-31 15:38:36 +00:00
2022-04-13 19:33:34 +00:00
mkdir -p /build/build_docker
cd /build/build_docker
rm -f CMakeCache.txt
2020-10-05 07:06:38 +00:00
# Read cmake arguments into array (possibly empty)
2020-10-04 10:23:25 +00:00
read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}"
env
2022-04-13 15:45:15 +00:00
2022-04-21 11:43:24 +00:00
if [ -n "$MAKE_DEB" ]; then
rm -rf /build/packages/root
# NOTE: this is for backward compatibility with previous releases,
# that does not diagnostics tool (only script).
if [ -d /build/programs/diagnostics ]; then
if [ -z "$SANITIZER" ]; then
# We need to check if clickhouse-diagnostics is fine and build it
(
cd /build/programs/diagnostics
make test-no-docker
GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build
mv clickhouse-diagnostics ..
)
else
echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics
chmod +x /build/programs/clickhouse-diagnostics
fi
fi
2022-04-21 11:43:24 +00:00
fi
2022-04-13 19:33:34 +00:00
ccache_status
2022-04-21 15:18:43 +00:00
# clear cache stats
ccache --zero-stats ||:
2022-04-13 15:45:15 +00:00
if [ "$BUILD_MUSL_KEEPER" == "1" ]
then
2022-04-21 12:15:24 +00:00
# build keeper with musl separately
# and without rust bindings
cmake --debug-trycompile -DENABLE_RUST=OFF -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
2022-04-13 15:45:15 +00:00
# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
ninja $NINJA_FLAGS clickhouse-keeper
2022-04-14 13:43:36 +00:00
2022-04-14 10:23:09 +00:00
ls -la ./programs/
ldd ./programs/clickhouse-keeper
2022-04-14 13:43:36 +00:00
if [ -n "$MAKE_DEB" ]; then
# No quotes because I want it to expand to nothing if empty.
# shellcheck disable=SC2086
2022-04-21 14:11:43 +00:00
DESTDIR=/build/packages/root ninja $NINJA_FLAGS programs/keeper/install
2022-04-14 13:43:36 +00:00
fi
2022-04-21 12:15:24 +00:00
rm -f CMakeCache.txt
2022-04-13 15:45:15 +00:00
2022-04-21 12:15:24 +00:00
# Build the rest of binaries
cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
2022-04-14 13:44:09 +00:00
else
2022-04-21 12:15:24 +00:00
# Build everything
cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
2022-04-14 13:43:36 +00:00
fi
2021-03-30 11:31:39 +00:00
if [ "coverity" == "$COMBINED_OUTPUT" ]
then
mkdir -p /workdir/cov-analysis
2022-04-07 16:21:13 +00:00
wget --post-data "token=$COVERITY_TOKEN&project=ClickHouse%2FClickHouse" -qO- https://scan.coverity.com/download/linux64 | tar xz -C /workdir/cov-analysis --strip-components 1
export PATH=$PATH:/workdir/cov-analysis/bin
cov-configure --config ./coverity.config --template --comptype clangcc --compiler "$CC"
SCAN_WRAPPER="cov-build --config ./coverity.config --dir cov-int"
fi
2022-02-08 18:12:04 +00:00
# No quotes because I want it to expand to nothing if empty.
# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
$SCAN_WRAPPER ninja $NINJA_FLAGS $BUILD_TARGET
2021-03-31 15:43:31 +00:00
2022-04-14 10:23:09 +00:00
ls -la ./programs
ccache_status
2022-02-08 18:12:04 +00:00
if [ -n "$MAKE_DEB" ]; then
# No quotes because I want it to expand to nothing if empty.
# shellcheck disable=SC2086
DESTDIR=/build/packages/root ninja $NINJA_FLAGS programs/install
cp /build/programs/clickhouse-diagnostics /build/packages/root/usr/bin
cp /build/programs/clickhouse-diagnostics /output
2022-02-08 18:12:04 +00:00
bash -x /build/packages/build
fi
2021-03-31 15:43:31 +00:00
mv ./programs/clickhouse* /output
2022-08-09 06:40:17 +00:00
[ -x ./programs/self-extracting/clickhouse ] && mv ./programs/self-extracting/clickhouse /output
2020-09-17 14:01:48 +00:00
mv ./src/unit_tests_dbms /output ||: # may not exist for some binary builds
2019-09-20 20:15:42 +00:00
prepare_combined_output () {
local OUTPUT
OUTPUT="$1"
mkdir -p "$OUTPUT"/config
cp /build/programs/server/config.xml "$OUTPUT"/config
cp /build/programs/server/users.xml "$OUTPUT"/config
cp -r --dereference /build/programs/server/config.d "$OUTPUT"/config
}
2019-12-25 16:08:44 +00:00
# Different files for performance test.
if [ "$WITH_PERFORMANCE" == 1 ]
2019-12-25 16:08:44 +00:00
then
PERF_OUTPUT=/workdir/performance/output
mkdir -p "$PERF_OUTPUT"
cp -r ../tests/performance "$PERF_OUTPUT"
cp -r ../tests/config/top_level_domains "$PERF_OUTPUT"
cp -r ../docker/test/performance-comparison/config "$PERF_OUTPUT" ||:
for SRC in /output/clickhouse*; do
# Copy all clickhouse* files except packages and bridges
[[ "$SRC" != *.* ]] && [[ "$SRC" != *-bridge ]] && \
cp -d "$SRC" "$PERF_OUTPUT"
done
if [ -x "$PERF_OUTPUT"/clickhouse-keeper ]; then
# Replace standalone keeper by symlink
ln -sf clickhouse "$PERF_OUTPUT"/clickhouse-keeper
fi
2020-03-11 14:29:34 +00:00
cp -r ../docker/test/performance-comparison "$PERF_OUTPUT"/scripts ||:
prepare_combined_output "$PERF_OUTPUT"
# We have to know the revision that corresponds to this binary build.
# It is not the nominal SHA from pull/*/head, but the pull/*/merge, which is
# head merged to master by github, at some point after the PR is updated.
# There are some quirks to consider:
# - apparently the real SHA is not recorded in system.build_options;
# - it can change at any time as github pleases, so we can't just record
# the SHA and use it later, it might become inaccessible;
# - CI has an immutable snapshot of repository that it uses for all checks
# for a given nominal SHA, but it is not accessible outside Yandex.
# This is why we add this repository snapshot from CI to the performance test
# package.
mkdir "$PERF_OUTPUT"/ch
git -C "$PERF_OUTPUT"/ch init --bare
git -C "$PERF_OUTPUT"/ch remote add origin /build
git -C "$PERF_OUTPUT"/ch fetch --no-tags --depth 50 origin HEAD:pr
git -C "$PERF_OUTPUT"/ch fetch --no-tags --depth 50 origin master:master
git -C "$PERF_OUTPUT"/ch reset --soft pr
git -C "$PERF_OUTPUT"/ch log -5
(
cd "$PERF_OUTPUT"/..
2023-01-01 20:17:43 +00:00
tar -cv --zstd -f /output/performance.tar.zst output
)
2019-12-25 16:08:44 +00:00
fi
2023-01-02 01:06:11 +00:00
# May be set for performance test.
if [ "" != "$COMBINED_OUTPUT" ]
2019-09-20 20:15:42 +00:00
then
prepare_combined_output /output
2023-01-01 20:17:43 +00:00
tar -cv --zstd -f "$COMBINED_OUTPUT.tar.zst" /output
2019-09-20 20:15:42 +00:00
rm -r /output/*
2023-01-01 20:17:43 +00:00
mv "$COMBINED_OUTPUT.tar.zst" /output
2019-09-20 20:15:42 +00:00
fi
2021-03-31 15:43:31 +00:00
if [ "coverity" == "$COMBINED_OUTPUT" ]
then
2023-01-01 20:17:43 +00:00
tar -cv --zstd -f "coverity-scan.tar.zst" cov-int
mv "coverity-scan.tar.zst" /output
fi
ccache_status
ccache --evict-older-than 1d
2021-08-24 00:09:19 +00:00
2021-03-31 23:51:47 +00:00
if [ "${CCACHE_DEBUG:-}" == "1" ]
2021-03-31 21:54:45 +00:00
then
2021-04-01 12:46:55 +00:00
find . -name '*.ccache-*' -print0 \
2021-04-01 18:02:35 +00:00
| tar -c -I pixz -f /output/ccache-debug.txz --null -T -
2021-03-31 21:54:45 +00:00
fi
2021-03-31 15:43:31 +00:00
2021-04-01 12:12:30 +00:00
if [ -n "$CCACHE_LOGFILE" ]
2021-03-31 21:54:45 +00:00
then
# Compress the log as well, or else the CI will try to compress all log
# files in place, and will fail because this directory is not writable.
2021-04-01 17:00:08 +00:00
tar -cv -I pixz -f /output/ccache.log.txz "$CCACHE_LOGFILE"
2021-03-31 21:54:45 +00:00
fi
2022-04-22 11:37:08 +00:00
ls -l /output