ClickHouse/tests/docker_scripts/fasttest_runner.sh

339 lines
10 KiB
Bash
Raw Normal View History

2020-07-08 13:06:35 +00:00
#!/bin/bash
set -xeu
set -o pipefail
trap "exit" INT TERM
trap 'kill $(jobs -pr) ||:' EXIT
# This script is separated into two stages, cloning and everything else, so
2020-09-07 10:19:01 +00:00
# that we can run the "everything else" stage from the cloned source.
stage=${stage:-}
2021-04-01 18:05:09 +00:00
# Compiler version, normally set by Dockerfile
2024-07-16 01:58:55 +00:00
export LLVM_VERSION=${LLVM_VERSION:-18}
# A variable to pass additional flags to CMake.
# Here we explicitly default it to nothing so that bash doesn't complain about
# it being undefined. Also read it as array so that we can pass an empty list
# of additional variable to cmake properly, and it doesn't generate an extra
# empty parameter.
2023-12-09 20:54:09 +00:00
# Read it as CMAKE_FLAGS to not lose exported FASTTEST_CMAKE_FLAGS on subsequent launch
2023-03-27 13:35:51 +00:00
read -ra CMAKE_FLAGS <<< "${FASTTEST_CMAKE_FLAGS:-}"
2020-07-08 13:06:35 +00:00
# Run only matching tests.
FASTTEST_FOCUS=${FASTTEST_FOCUS:-""}
2020-09-29 18:01:49 +00:00
FASTTEST_WORKSPACE=$(readlink -f "${FASTTEST_WORKSPACE:-.}")
FASTTEST_SOURCE=$(readlink -f "${FASTTEST_SOURCE:-$FASTTEST_WORKSPACE/ch}")
FASTTEST_BUILD=$(readlink -f "${FASTTEST_BUILD:-${BUILD:-$FASTTEST_WORKSPACE/build}}")
FASTTEST_DATA=$(readlink -f "${FASTTEST_DATA:-$FASTTEST_WORKSPACE/db-fasttest}")
FASTTEST_OUTPUT=$(readlink -f "${FASTTEST_OUTPUT:-$FASTTEST_WORKSPACE}")
2020-10-22 17:30:50 +00:00
PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH"
2023-09-20 17:07:42 +00:00
# Work around for non-existent user
if [ "$HOME" == "/" ]; then
HOME="$FASTTEST_WORKSPACE/user-home"
mkdir -p "$HOME"
export HOME
fi
2020-09-29 18:01:49 +00:00
2020-09-30 17:39:38 +00:00
# Export these variables, so that all subsequent invocations of the script
# use them, and not try to guess them anew, which leads to weird effects.
export FASTTEST_WORKSPACE
export FASTTEST_SOURCE
export FASTTEST_BUILD
export FASTTEST_DATA
2024-08-05 17:41:47 +00:00
export FASTTEST_OUTPUT
2020-10-22 17:30:50 +00:00
export PATH
2020-09-30 17:39:38 +00:00
function ccache_status
{
ccache --show-config ||:
ccache --show-stats ||:
SCCACHE_NO_DAEMON=1 sccache --show-stats ||:
}
2020-09-29 18:01:49 +00:00
function start_server
{
2020-09-29 18:01:49 +00:00
set -m # Spawn server in its own process groups
2020-12-08 20:58:27 +00:00
local opts=(
2020-12-28 11:46:53 +00:00
--config-file "$FASTTEST_DATA/config.xml"
--pid-file "$FASTTEST_DATA/clickhouse-server.pid"
2020-12-08 20:58:27 +00:00
--
--path "$FASTTEST_DATA"
--user_files_path "$FASTTEST_DATA/user_files"
--top_level_domains_path "$FASTTEST_DATA/top_level_domains"
2021-05-12 20:00:28 +00:00
--keeper_server.storage_path "$FASTTEST_DATA/coordination"
2020-12-08 20:58:27 +00:00
)
clickhouse-server "${opts[@]}" &>> "$FASTTEST_OUTPUT/server.log" &
2020-09-29 18:01:49 +00:00
set +m
for _ in {1..60}; do
if clickhouse-client --query "select 1"; then
break
fi
sleep 1
done
if ! clickhouse-client --query "select 1"; then
echo "Failed to wait until ClickHouse server starts."
2020-09-29 18:01:49 +00:00
return 1
fi
local server_pid
server_pid="$(cat "$FASTTEST_DATA/clickhouse-server.pid")"
2020-09-29 18:01:49 +00:00
echo "ClickHouse server pid '$server_pid' started and responded"
}
2024-07-03 21:13:50 +00:00
export -f start_server
function clone_root
{
[ "$UID" -eq 0 ] && git config --global --add safe.directory "$FASTTEST_SOURCE"
2021-04-01 21:04:59 +00:00
git clone --depth 1 https://github.com/ClickHouse/ClickHouse.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt"
(
cd "$FASTTEST_SOURCE"
if [ "$PULL_REQUEST_NUMBER" != "0" ]; then
2021-04-01 18:44:43 +00:00
if git fetch --depth 1 origin "+refs/pull/$PULL_REQUEST_NUMBER/merge"; then
git checkout FETCH_HEAD
2021-04-01 20:28:53 +00:00
echo "Checked out pull/$PULL_REQUEST_NUMBER/merge ($(git rev-parse FETCH_HEAD))"
else
2021-04-01 18:44:43 +00:00
git fetch --depth 1 origin "+refs/pull/$PULL_REQUEST_NUMBER/head"
git checkout "$COMMIT_SHA"
2021-04-01 20:28:53 +00:00
echo "Checked out nominal SHA $COMMIT_SHA for PR $PULL_REQUEST_NUMBER"
fi
else
if [ -v COMMIT_SHA ]; then
2021-04-01 18:44:43 +00:00
git fetch --depth 1 origin "$COMMIT_SHA"
git checkout "$COMMIT_SHA"
2021-04-01 20:28:53 +00:00
echo "Checked out nominal SHA $COMMIT_SHA for master"
else
echo "Using default repository head $(git rev-parse HEAD)"
fi
fi
)
}
2020-07-08 16:02:19 +00:00
2020-09-29 18:01:49 +00:00
function clone_submodules
{
(
cd "$FASTTEST_SOURCE"
2020-07-08 16:02:19 +00:00
SUBMODULES_TO_UPDATE=(
2021-10-12 00:34:07 +00:00
contrib/sysroot
2021-09-06 15:59:46 +00:00
contrib/magic_enum
2021-02-27 18:02:44 +00:00
contrib/abseil-cpp
contrib/boost
contrib/zlib-ng
contrib/libxml2
contrib/libunwind
contrib/fmtlib
contrib/aklomp-base64
contrib/cctz
contrib/libcpuid
contrib/libdivide
contrib/double-conversion
contrib/llvm-project
contrib/lz4
contrib/zstd
contrib/fastops
contrib/rapidjson
contrib/re2
contrib/sparsehash-c11
contrib/croaring
contrib/miniselect
contrib/xz
2020-12-05 21:32:03 +00:00
contrib/dragonbox
2020-12-05 17:09:12 +00:00
contrib/fast_float
2021-01-25 15:32:59 +00:00
contrib/NuRaft
contrib/jemalloc
contrib/replxx
2022-04-21 16:36:47 +00:00
contrib/wyhash
2022-07-11 15:43:56 +00:00
contrib/c-ares
2022-09-25 17:58:00 +00:00
contrib/morton-nd
contrib/xxHash
2024-04-15 14:06:59 +00:00
contrib/expected
contrib/simdjson
2022-06-27 08:38:00 +00:00
contrib/liburing
2023-05-10 11:56:17 +00:00
contrib/libfiu
2023-07-23 15:30:23 +00:00
contrib/incbin
2023-08-06 00:52:36 +00:00
contrib/yaml-cpp
)
git submodule sync
2023-09-20 16:10:50 +00:00
git submodule init
2024-05-20 04:13:06 +00:00
# Network is unreliable
for _ in {1..10}
do
# --jobs does not work as fast as real parallel running
printf '%s\0' "${SUBMODULES_TO_UPDATE[@]}" | \
xargs --max-procs=100 --null --no-run-if-empty --max-args=1 \
git submodule update --depth 1 --single-branch && break
sleep 1
done
git submodule foreach git reset --hard
git submodule foreach git checkout @ -f
git submodule foreach git clean -xfd
)
2020-09-29 18:01:49 +00:00
}
2020-09-29 18:55:06 +00:00
function run_cmake
2020-09-29 18:01:49 +00:00
{
CMAKE_LIBS_CONFIG=(
"-DENABLE_LIBRARIES=0"
"-DENABLE_TESTS=0"
"-DENABLE_UTILS=0"
"-DENABLE_THINLTO=0"
2021-01-25 15:53:13 +00:00
"-DENABLE_NURAFT=1"
"-DENABLE_SIMDJSON=1"
"-DENABLE_JEMALLOC=1"
2022-06-27 08:38:00 +00:00
"-DENABLE_LIBURING=1"
2023-08-06 00:52:36 +00:00
"-DENABLE_YAML_CPP=1"
)
export CCACHE_DIR="$FASTTEST_WORKSPACE/ccache"
export CCACHE_COMPRESSLEVEL=5
export CCACHE_BASEDIR="$FASTTEST_SOURCE"
export CCACHE_NOHASHDIR=true
export CCACHE_COMPILERCHECK=content
export CCACHE_MAXSIZE=15G
ccache_status
ccache --zero-stats ||:
mkdir "$FASTTEST_BUILD" ||:
(
cd "$FASTTEST_BUILD"
2023-12-09 20:54:09 +00:00
cmake "$FASTTEST_SOURCE" -DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" -DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" -DCMAKE_TOOLCHAIN_FILE="${FASTTEST_SOURCE}/cmake/linux/toolchain-x86_64-musl.cmake" "${CMAKE_LIBS_CONFIG[@]}" "${CMAKE_FLAGS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/cmake_log.txt"
)
2020-09-29 18:55:06 +00:00
}
2020-07-08 13:06:35 +00:00
2020-09-29 18:55:06 +00:00
function build
{
(
cd "$FASTTEST_BUILD"
TIMEFORMAT=$'\nreal\t%3R\nuser\t%3U\nsys\t%3S'
2023-11-17 12:50:13 +00:00
( time ninja clickhouse-bundle clickhouse-stripped) |& ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/build_log.txt"
BUILD_SECONDS_ELAPSED=$(awk '/^....-..-.. ..:..:.. real\t[0-9]/ {print $4}' < "$FASTTEST_OUTPUT/build_log.txt")
echo "build_clickhouse_fasttest_binary: [ OK ] $BUILD_SECONDS_ELAPSED sec." \
| ts '%Y-%m-%d %H:%M:%S' \
| tee "$FASTTEST_OUTPUT/test_result.txt"
(
# This query should fail, and print stacktrace with proper symbol names (even on a stripped binary)
clickhouse_output=$(programs/clickhouse-stripped --stacktrace -q 'select' 2>&1 || :)
if [[ $clickhouse_output =~ DB::LocalServer::main ]]; then
echo "stripped_clickhouse_shows_symbols_names: [ OK ] 0 sec."
else
echo -e "stripped_clickhouse_shows_symbols_names: [ FAIL ] 0 sec. - clickhouse output:\n\n$clickhouse_output\n"
fi
) | ts '%Y-%m-%d %H:%M:%S' | tee -a "$FASTTEST_OUTPUT/test_result.txt"
if [ "$COPY_CLICKHOUSE_BINARY_TO_OUTPUT" -eq "1" ]; then
mkdir -p "$FASTTEST_OUTPUT/binaries/"
cp programs/clickhouse "$FASTTEST_OUTPUT/binaries/clickhouse"
zstd --threads=0 programs/clickhouse-stripped -o "$FASTTEST_OUTPUT/binaries/clickhouse-stripped.zst"
fi
ccache_status
ccache --evict-older-than 1d ||:
)
2020-09-29 18:01:49 +00:00
}
2020-07-08 13:06:35 +00:00
2020-09-29 18:01:49 +00:00
function configure
{
clickhouse-client --version
clickhouse-test --help
mkdir -p "$FASTTEST_DATA"{,/client-config}
cp -a "$FASTTEST_SOURCE/programs/server/"{config,users}.xml "$FASTTEST_DATA"
"$FASTTEST_SOURCE/tests/config/install.sh" "$FASTTEST_DATA" "$FASTTEST_DATA/client-config"
cp -a "$FASTTEST_SOURCE/programs/server/config.d/log_to_console.xml" "$FASTTEST_DATA/config.d"
# doesn't support SSL
rm -f "$FASTTEST_DATA/config.d/secure_ports.xml"
2020-09-29 18:01:49 +00:00
}
2020-09-29 18:01:49 +00:00
function run_tests
{
clickhouse-server --version
clickhouse-test --help
2020-07-28 11:16:06 +00:00
2020-09-29 18:01:49 +00:00
start_server
2020-07-29 11:39:08 +00:00
set +e
local NPROC
NPROC=$(nproc)
NPROC=$((NPROC / 2))
if [[ $NPROC == 0 ]]; then
NPROC=1
fi
export CLICKHOUSE_CONFIG_DIR=$FASTTEST_DATA
export CLICKHOUSE_CONFIG="$FASTTEST_DATA/config.xml"
export CLICKHOUSE_USER_FILES="$FASTTEST_DATA/user_files"
export CLICKHOUSE_SCHEMA_FILES="$FASTTEST_DATA/format_schemas"
local test_opts=(
--hung-check
--fast-tests-only
2022-03-28 12:56:23 +00:00
--no-random-settings
2022-10-28 23:26:06 +00:00
--no-random-merge-tree-settings
--no-long
--testname
--shard
--zookeeper
--check-zookeeper-session
--order random
--print-time
2023-01-25 15:06:40 +00:00
--report-logs-stats
--jobs "${NPROC}"
)
time clickhouse-test "${test_opts[@]}" -- "$FASTTEST_FOCUS" 2>&1 \
2021-05-20 12:31:33 +00:00
| ts '%Y-%m-%d %H:%M:%S' \
| tee -a "$FASTTEST_OUTPUT/test_result.txt"
set -e
clickhouse stop --pid-path "$FASTTEST_DATA"
}
2024-07-03 17:39:00 +00:00
export -f run_tests
case "$stage" in
"")
2020-09-07 10:19:01 +00:00
ls -la
;&
"clone_root")
clone_root
2020-09-29 18:01:49 +00:00
;&
"clone_submodules")
2021-04-01 21:02:35 +00:00
clone_submodules 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/submodule_log.txt"
2020-09-29 18:01:49 +00:00
;&
2020-09-29 18:55:06 +00:00
"run_cmake")
2023-09-20 16:02:59 +00:00
cd "$FASTTEST_WORKSPACE"
2020-09-29 18:55:06 +00:00
run_cmake
;&
2020-09-29 18:01:49 +00:00
"build")
build
;&
"configure")
2020-09-29 20:09:52 +00:00
# The `install_log.txt` is also needed for compatibility with old CI task --
# if there is no log, it will decide that build failed.
2021-04-01 21:02:35 +00:00
configure 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/install_log.txt"
2020-09-29 18:01:49 +00:00
;&
"run_tests")
run_tests ||:
2024-08-09 14:22:33 +00:00
/repo/tests/docker_scripts/process_functional_tests_result.py --in-results-dir "$FASTTEST_OUTPUT/" \
--out-results-file "$FASTTEST_OUTPUT/test_results.tsv" \
--out-status-file "$FASTTEST_OUTPUT/check_status.tsv" || echo -e "failure\tCannot parse results" > "$FASTTEST_OUTPUT/check_status.tsv"
2020-10-14 14:12:04 +00:00
;;
*)
echo "Unknown test stage '$stage'"
exit 1
esac
pstree -apgT
jobs