mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-12-12 09:22:05 +00:00
356 lines
11 KiB
Bash
Executable File
356 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
set -xeu
|
|
set -o pipefail
|
|
trap "exit" INT TERM
|
|
trap 'kill $(jobs -pr) ||:' EXIT
|
|
|
|
# This script is separated into two stages, cloning and everything else, so
|
|
# that we can run the "everything else" stage from the cloned source.
|
|
stage=${stage:-}
|
|
|
|
# Compiler version, normally set by Dockerfile
|
|
export LLVM_VERSION=${LLVM_VERSION:-18}
|
|
|
|
# For some reason it is needed for tests with 'expect' but I don't know, why
|
|
groupadd --system --gid 1000 clickhouse
|
|
useradd --system --gid 1000 --uid 1000 clickhouse
|
|
|
|
# A variable to pass additional flags to CMake.
|
|
# Here we explicitly default it to nothing so that bash doesn't complain about
|
|
# it being undefined. Also read it as array so that we can pass an empty list
|
|
# of additional variable to cmake properly, and it doesn't generate an extra
|
|
# empty parameter.
|
|
# Read it as CMAKE_FLAGS to not lose exported FASTTEST_CMAKE_FLAGS on subsequent launch
|
|
read -ra CMAKE_FLAGS <<< "${FASTTEST_CMAKE_FLAGS:-}"
|
|
|
|
# Run only matching tests.
|
|
FASTTEST_FOCUS=${FASTTEST_FOCUS:-""}
|
|
|
|
FASTTEST_WORKSPACE=$(readlink -f "${FASTTEST_WORKSPACE:-.}")
|
|
FASTTEST_SOURCE=$(readlink -f "${FASTTEST_SOURCE:-$FASTTEST_WORKSPACE/ch}")
|
|
FASTTEST_BUILD=$(readlink -f "${FASTTEST_BUILD:-${BUILD:-$FASTTEST_WORKSPACE/build}}")
|
|
FASTTEST_DATA=$(readlink -f "${FASTTEST_DATA:-$FASTTEST_WORKSPACE/db-fasttest}")
|
|
FASTTEST_OUTPUT=$(readlink -f "${FASTTEST_OUTPUT:-$FASTTEST_WORKSPACE}")
|
|
PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH"
|
|
# Work around for non-existent user
|
|
if [ "$HOME" == "/" ]; then
|
|
HOME="$FASTTEST_WORKSPACE/user-home"
|
|
mkdir -p "$HOME"
|
|
export HOME
|
|
fi
|
|
|
|
# Export these variables, so that all subsequent invocations of the script
|
|
# use them, and not try to guess them anew, which leads to weird effects.
|
|
export FASTTEST_WORKSPACE
|
|
export FASTTEST_SOURCE
|
|
export FASTTEST_BUILD
|
|
export FASTTEST_DATA
|
|
export FASTTEST_OUT
|
|
export PATH
|
|
|
|
function ccache_status
|
|
{
|
|
ccache --show-config ||:
|
|
ccache --show-stats ||:
|
|
SCCACHE_NO_DAEMON=1 sccache --show-stats ||:
|
|
}
|
|
|
|
function start_server
|
|
{
|
|
set -m # Spawn server in its own process groups
|
|
|
|
local opts=(
|
|
--config-file "$FASTTEST_DATA/config.xml"
|
|
--pid-file "$FASTTEST_DATA/clickhouse-server.pid"
|
|
--
|
|
--path "$FASTTEST_DATA"
|
|
--user_files_path "$FASTTEST_DATA/user_files"
|
|
--top_level_domains_path "$FASTTEST_DATA/top_level_domains"
|
|
--keeper_server.storage_path "$FASTTEST_DATA/coordination"
|
|
)
|
|
clickhouse-server "${opts[@]}" &>> "$FASTTEST_OUTPUT/server.log" &
|
|
set +m
|
|
|
|
for _ in {1..60}; do
|
|
if clickhouse-client --query "select 1"; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
if ! clickhouse-client --query "select 1"; then
|
|
echo "Failed to wait until ClickHouse server starts."
|
|
return 1
|
|
fi
|
|
|
|
local server_pid
|
|
server_pid="$(cat "$FASTTEST_DATA/clickhouse-server.pid")"
|
|
echo "ClickHouse server pid '$server_pid' started and responded"
|
|
}
|
|
|
|
export -f start_server
|
|
|
|
function clone_root
|
|
{
|
|
[ "$UID" -eq 0 ] && git config --global --add safe.directory "$FASTTEST_SOURCE"
|
|
git clone --depth 1 https://github.com/ClickHouse/ClickHouse.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt"
|
|
|
|
(
|
|
cd "$FASTTEST_SOURCE"
|
|
if [ "$PULL_REQUEST_NUMBER" != "0" ]; then
|
|
if git fetch --depth 1 origin "+refs/pull/$PULL_REQUEST_NUMBER/merge"; then
|
|
git checkout FETCH_HEAD
|
|
echo "Checked out pull/$PULL_REQUEST_NUMBER/merge ($(git rev-parse FETCH_HEAD))"
|
|
else
|
|
git fetch --depth 1 origin "+refs/pull/$PULL_REQUEST_NUMBER/head"
|
|
git checkout "$COMMIT_SHA"
|
|
echo "Checked out nominal SHA $COMMIT_SHA for PR $PULL_REQUEST_NUMBER"
|
|
fi
|
|
else
|
|
if [ -v COMMIT_SHA ]; then
|
|
git fetch --depth 1 origin "$COMMIT_SHA"
|
|
git checkout "$COMMIT_SHA"
|
|
echo "Checked out nominal SHA $COMMIT_SHA for master"
|
|
else
|
|
echo "Using default repository head $(git rev-parse HEAD)"
|
|
fi
|
|
fi
|
|
)
|
|
}
|
|
|
|
function clone_submodules
|
|
{
|
|
(
|
|
cd "$FASTTEST_SOURCE"
|
|
|
|
SUBMODULES_TO_UPDATE=(
|
|
contrib/sysroot
|
|
contrib/magic_enum
|
|
contrib/abseil-cpp
|
|
contrib/boost
|
|
contrib/zlib-ng
|
|
contrib/libxml2
|
|
contrib/libunwind
|
|
contrib/fmtlib
|
|
contrib/aklomp-base64
|
|
contrib/cctz
|
|
contrib/libcpuid
|
|
contrib/libdivide
|
|
contrib/double-conversion
|
|
contrib/llvm-project
|
|
contrib/lz4
|
|
contrib/zstd
|
|
contrib/fastops
|
|
contrib/rapidjson
|
|
contrib/re2
|
|
contrib/sparsehash-c11
|
|
contrib/croaring
|
|
contrib/miniselect
|
|
contrib/xz
|
|
contrib/dragonbox
|
|
contrib/fast_float
|
|
contrib/NuRaft
|
|
contrib/jemalloc
|
|
contrib/replxx
|
|
contrib/wyhash
|
|
contrib/c-ares
|
|
contrib/morton-nd
|
|
contrib/xxHash
|
|
contrib/expected
|
|
contrib/simdjson
|
|
contrib/liburing
|
|
contrib/libfiu
|
|
contrib/incbin
|
|
contrib/yaml-cpp
|
|
)
|
|
|
|
git submodule sync
|
|
git submodule init
|
|
|
|
# Network is unreliable
|
|
for _ in {1..10}
|
|
do
|
|
# --jobs does not work as fast as real parallel running
|
|
printf '%s\0' "${SUBMODULES_TO_UPDATE[@]}" | \
|
|
xargs --max-procs=100 --null --no-run-if-empty --max-args=1 \
|
|
git submodule update --depth 1 --single-branch && break
|
|
sleep 1
|
|
done
|
|
|
|
git submodule foreach git reset --hard
|
|
git submodule foreach git checkout @ -f
|
|
git submodule foreach git clean -xfd
|
|
)
|
|
}
|
|
|
|
function run_cmake
|
|
{
|
|
CMAKE_LIBS_CONFIG=(
|
|
"-DENABLE_LIBRARIES=0"
|
|
"-DENABLE_TESTS=0"
|
|
"-DENABLE_UTILS=0"
|
|
"-DENABLE_THINLTO=0"
|
|
"-DENABLE_NURAFT=1"
|
|
"-DENABLE_SIMDJSON=1"
|
|
"-DENABLE_JEMALLOC=1"
|
|
"-DENABLE_LIBURING=1"
|
|
"-DENABLE_YAML_CPP=1"
|
|
)
|
|
|
|
export CCACHE_DIR="$FASTTEST_WORKSPACE/ccache"
|
|
export CCACHE_COMPRESSLEVEL=5
|
|
export CCACHE_BASEDIR="$FASTTEST_SOURCE"
|
|
export CCACHE_NOHASHDIR=true
|
|
export CCACHE_COMPILERCHECK=content
|
|
export CCACHE_MAXSIZE=15G
|
|
|
|
ccache_status
|
|
ccache --zero-stats ||:
|
|
|
|
mkdir "$FASTTEST_BUILD" ||:
|
|
|
|
(
|
|
cd "$FASTTEST_BUILD"
|
|
cmake "$FASTTEST_SOURCE" -DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" -DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" -DCMAKE_TOOLCHAIN_FILE="${FASTTEST_SOURCE}/cmake/linux/toolchain-x86_64-musl.cmake" "${CMAKE_LIBS_CONFIG[@]}" "${CMAKE_FLAGS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/cmake_log.txt"
|
|
)
|
|
}
|
|
|
|
function build
|
|
{
|
|
(
|
|
cd "$FASTTEST_BUILD"
|
|
TIMEFORMAT=$'\nreal\t%3R\nuser\t%3U\nsys\t%3S'
|
|
( time ninja clickhouse-bundle clickhouse-stripped) |& ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/build_log.txt"
|
|
BUILD_SECONDS_ELAPSED=$(awk '/^....-..-.. ..:..:.. real\t[0-9]/ {print $4}' < "$FASTTEST_OUTPUT/build_log.txt")
|
|
echo "build_clickhouse_fasttest_binary: [ OK ] $BUILD_SECONDS_ELAPSED sec." \
|
|
| ts '%Y-%m-%d %H:%M:%S' \
|
|
| tee "$FASTTEST_OUTPUT/test_result.txt"
|
|
|
|
(
|
|
# This query should fail, and print stacktrace with proper symbol names (even on a stripped binary)
|
|
clickhouse_output=$(programs/clickhouse-stripped --stacktrace -q 'select' 2>&1 || :)
|
|
if [[ $clickhouse_output =~ DB::LocalServer::main ]]; then
|
|
echo "stripped_clickhouse_shows_symbols_names: [ OK ] 0 sec."
|
|
else
|
|
echo -e "stripped_clickhouse_shows_symbols_names: [ FAIL ] 0 sec. - clickhouse output:\n\n$clickhouse_output\n"
|
|
fi
|
|
) | ts '%Y-%m-%d %H:%M:%S' | tee -a "$FASTTEST_OUTPUT/test_result.txt"
|
|
|
|
if [ "$COPY_CLICKHOUSE_BINARY_TO_OUTPUT" -eq "1" ]; then
|
|
mkdir -p "$FASTTEST_OUTPUT/binaries/"
|
|
cp programs/clickhouse "$FASTTEST_OUTPUT/binaries/clickhouse"
|
|
|
|
zstd --threads=0 programs/clickhouse-stripped -o "$FASTTEST_OUTPUT/binaries/clickhouse-stripped.zst"
|
|
fi
|
|
ccache_status
|
|
ccache --evict-older-than 1d ||:
|
|
)
|
|
}
|
|
|
|
function configure
|
|
{
|
|
clickhouse-client --version
|
|
clickhouse-test --help
|
|
|
|
mkdir -p "$FASTTEST_DATA"{,/client-config}
|
|
cp -a "$FASTTEST_SOURCE/programs/server/"{config,users}.xml "$FASTTEST_DATA"
|
|
"$FASTTEST_SOURCE/tests/config/install.sh" "$FASTTEST_DATA" "$FASTTEST_DATA/client-config"
|
|
cp -a "$FASTTEST_SOURCE/programs/server/config.d/log_to_console.xml" "$FASTTEST_DATA/config.d"
|
|
# doesn't support SSL
|
|
rm -f "$FASTTEST_DATA/config.d/secure_ports.xml"
|
|
}
|
|
|
|
function timeout_with_logging() {
|
|
local exit_code=0
|
|
|
|
timeout -s TERM --preserve-status "${@}" || exit_code="${?}"
|
|
|
|
if [[ "${exit_code}" -eq "124" ]]
|
|
then
|
|
echo "The command 'timeout ${*}' has been killed by timeout"
|
|
fi
|
|
|
|
return $exit_code
|
|
}
|
|
|
|
function run_tests
|
|
{
|
|
clickhouse-server --version
|
|
clickhouse-test --help
|
|
|
|
start_server
|
|
|
|
set +e
|
|
local NPROC
|
|
NPROC=$(nproc)
|
|
NPROC=$((NPROC / 2))
|
|
if [[ $NPROC == 0 ]]; then
|
|
NPROC=1
|
|
fi
|
|
|
|
export CLICKHOUSE_CONFIG_DIR=$FASTTEST_DATA
|
|
export CLICKHOUSE_CONFIG="$FASTTEST_DATA/config.xml"
|
|
export CLICKHOUSE_USER_FILES="$FASTTEST_DATA/user_files"
|
|
export CLICKHOUSE_SCHEMA_FILES="$FASTTEST_DATA/format_schemas"
|
|
|
|
local test_opts=(
|
|
--hung-check
|
|
--fast-tests-only
|
|
--no-random-settings
|
|
--no-random-merge-tree-settings
|
|
--no-long
|
|
--testname
|
|
--shard
|
|
--zookeeper
|
|
--check-zookeeper-session
|
|
--order random
|
|
--print-time
|
|
--report-logs-stats
|
|
--jobs "${NPROC}"
|
|
)
|
|
time clickhouse-test "${test_opts[@]}" -- "$FASTTEST_FOCUS" 2>&1 \
|
|
| ts '%Y-%m-%d %H:%M:%S' \
|
|
| tee -a "$FASTTEST_OUTPUT/test_result.txt"
|
|
set -e
|
|
|
|
clickhouse stop --pid-path "$FASTTEST_DATA"
|
|
}
|
|
|
|
export -f run_tests
|
|
|
|
case "$stage" in
|
|
"")
|
|
ls -la
|
|
;&
|
|
"clone_root")
|
|
clone_root
|
|
;&
|
|
"clone_submodules")
|
|
clone_submodules 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/submodule_log.txt"
|
|
;&
|
|
"run_cmake")
|
|
cd "$FASTTEST_WORKSPACE"
|
|
run_cmake
|
|
;&
|
|
"build")
|
|
build
|
|
;&
|
|
"configure")
|
|
# The `install_log.txt` is also needed for compatibility with old CI task --
|
|
# if there is no log, it will decide that build failed.
|
|
configure 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/install_log.txt"
|
|
;&
|
|
"run_tests")
|
|
timeout_with_logging 35m bash -c run_tests ||:
|
|
/process_functional_tests_result.py --in-results-dir "$FASTTEST_OUTPUT/" \
|
|
--out-results-file "$FASTTEST_OUTPUT/test_results.tsv" \
|
|
--out-status-file "$FASTTEST_OUTPUT/check_status.tsv" || echo -e "failure\tCannot parse results" > "$FASTTEST_OUTPUT/check_status.tsv"
|
|
;;
|
|
*)
|
|
echo "Unknown test stage '$stage'"
|
|
exit 1
|
|
esac
|
|
|
|
pstree -apgT
|
|
jobs
|