#!/bin/bash set -xeu set -o pipefail trap "exit" INT TERM trap 'kill $(jobs -pr) ||:' EXIT # This script is separated into two stages, cloning and everything else, so # that we can run the "everything else" stage from the cloned source. stage=${stage:-} # Compiler version, normally set by Dockerfile export LLVM_VERSION=${LLVM_VERSION:-17} # A variable to pass additional flags to CMake. # Here we explicitly default it to nothing so that bash doesn't complain about # it being undefined. Also read it as array so that we can pass an empty list # of additional variable to cmake properly, and it doesn't generate an extra # empty parameter. # Read it as CMAKE_FLAGS to not lose exported FASTTEST_CMAKE_FLAGS on subsequent launch read -ra CMAKE_FLAGS <<< "${FASTTEST_CMAKE_FLAGS:-}" # Run only matching tests. FASTTEST_FOCUS=${FASTTEST_FOCUS:-""} FASTTEST_WORKSPACE=$(readlink -f "${FASTTEST_WORKSPACE:-.}") FASTTEST_SOURCE=$(readlink -f "${FASTTEST_SOURCE:-$FASTTEST_WORKSPACE/ch}") FASTTEST_BUILD=$(readlink -f "${FASTTEST_BUILD:-${BUILD:-$FASTTEST_WORKSPACE/build}}") FASTTEST_DATA=$(readlink -f "${FASTTEST_DATA:-$FASTTEST_WORKSPACE/db-fasttest}") FASTTEST_OUTPUT=$(readlink -f "${FASTTEST_OUTPUT:-$FASTTEST_WORKSPACE}") PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH" # Work around for non-existent user if [ "$HOME" == "/" ]; then HOME="$FASTTEST_WORKSPACE/user-home" mkdir -p "$HOME" export HOME fi # Export these variables, so that all subsequent invocations of the script # use them, and not try to guess them anew, which leads to weird effects. export FASTTEST_WORKSPACE export FASTTEST_SOURCE export FASTTEST_BUILD export FASTTEST_DATA export FASTTEST_OUT export PATH function ccache_status { ccache --show-config ||: ccache --show-stats ||: SCCACHE_NO_DAEMON=1 sccache --show-stats ||: } function start_server { set -m # Spawn server in its own process groups local opts=( --config-file "$FASTTEST_DATA/config.xml" --pid-file "$FASTTEST_DATA/clickhouse-server.pid" -- --path "$FASTTEST_DATA" --user_files_path "$FASTTEST_DATA/user_files" --top_level_domains_path "$FASTTEST_DATA/top_level_domains" --keeper_server.storage_path "$FASTTEST_DATA/coordination" ) clickhouse-server "${opts[@]}" &>> "$FASTTEST_OUTPUT/server.log" & set +m for _ in {1..60}; do if clickhouse-client --query "select 1"; then break fi sleep 1 done if ! clickhouse-client --query "select 1"; then echo "Failed to wait until ClickHouse server starts." return 1 fi local server_pid server_pid="$(cat "$FASTTEST_DATA/clickhouse-server.pid")" echo "ClickHouse server pid '$server_pid' started and responded" } export -f start_server function clone_root { [ "$UID" -eq 0 ] && git config --global --add safe.directory "$FASTTEST_SOURCE" git clone --depth 1 https://github.com/ClickHouse/ClickHouse.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" ( cd "$FASTTEST_SOURCE" if [ "$PULL_REQUEST_NUMBER" != "0" ]; then if git fetch --depth 1 origin "+refs/pull/$PULL_REQUEST_NUMBER/merge"; then git checkout FETCH_HEAD echo "Checked out pull/$PULL_REQUEST_NUMBER/merge ($(git rev-parse FETCH_HEAD))" else git fetch --depth 1 origin "+refs/pull/$PULL_REQUEST_NUMBER/head" git checkout "$COMMIT_SHA" echo "Checked out nominal SHA $COMMIT_SHA for PR $PULL_REQUEST_NUMBER" fi else if [ -v COMMIT_SHA ]; then git fetch --depth 1 origin "$COMMIT_SHA" git checkout "$COMMIT_SHA" echo "Checked out nominal SHA $COMMIT_SHA for master" else echo "Using default repository head $(git rev-parse HEAD)" fi fi ) } function clone_submodules { ( cd "$FASTTEST_SOURCE" SUBMODULES_TO_UPDATE=( contrib/sysroot contrib/magic_enum contrib/abseil-cpp contrib/boost contrib/zlib-ng contrib/libxml2 contrib/libunwind contrib/fmtlib contrib/aklomp-base64 contrib/cctz contrib/libcpuid contrib/libdivide contrib/double-conversion contrib/llvm-project contrib/lz4 contrib/zstd contrib/fastops contrib/rapidjson contrib/re2 contrib/sparsehash-c11 contrib/croaring contrib/miniselect contrib/xz contrib/dragonbox contrib/fast_float contrib/NuRaft contrib/jemalloc contrib/replxx contrib/wyhash contrib/c-ares contrib/morton-nd contrib/xxHash contrib/expected contrib/simdjson contrib/liburing contrib/libfiu contrib/incbin contrib/yaml-cpp ) git submodule sync git submodule init # Network is unreliable for _ in {1..10} do # --jobs does not work as fast as real parallel running printf '%s\0' "${SUBMODULES_TO_UPDATE[@]}" | \ xargs --max-procs=100 --null --no-run-if-empty --max-args=1 \ git submodule update --depth 1 --single-branch && break sleep 1 done git submodule foreach git reset --hard git submodule foreach git checkout @ -f git submodule foreach git clean -xfd ) } function run_cmake { CMAKE_LIBS_CONFIG=( "-DENABLE_LIBRARIES=0" "-DENABLE_TESTS=0" "-DENABLE_UTILS=0" "-DENABLE_THINLTO=0" "-DENABLE_NURAFT=1" "-DENABLE_SIMDJSON=1" "-DENABLE_JEMALLOC=1" "-DENABLE_LIBURING=1" "-DENABLE_YAML_CPP=1" ) export CCACHE_DIR="$FASTTEST_WORKSPACE/ccache" export CCACHE_COMPRESSLEVEL=5 export CCACHE_BASEDIR="$FASTTEST_SOURCE" export CCACHE_NOHASHDIR=true export CCACHE_COMPILERCHECK=content export CCACHE_MAXSIZE=15G ccache_status ccache --zero-stats ||: mkdir "$FASTTEST_BUILD" ||: ( cd "$FASTTEST_BUILD" cmake "$FASTTEST_SOURCE" -DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" -DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" -DCMAKE_TOOLCHAIN_FILE="${FASTTEST_SOURCE}/cmake/linux/toolchain-x86_64-musl.cmake" "${CMAKE_LIBS_CONFIG[@]}" "${CMAKE_FLAGS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/cmake_log.txt" ) } function build { ( cd "$FASTTEST_BUILD" TIMEFORMAT=$'\nreal\t%3R\nuser\t%3U\nsys\t%3S' ( time ninja clickhouse-bundle clickhouse-stripped) |& ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/build_log.txt" BUILD_SECONDS_ELAPSED=$(awk '/^....-..-.. ..:..:.. real\t[0-9]/ {print $4}' < "$FASTTEST_OUTPUT/build_log.txt") echo "build_clickhouse_fasttest_binary: [ OK ] $BUILD_SECONDS_ELAPSED sec." \ | ts '%Y-%m-%d %H:%M:%S' \ | tee "$FASTTEST_OUTPUT/test_result.txt" ( # This query should fail, and print stacktrace with proper symbol names (even on a stripped binary) clickhouse_output=$(programs/clickhouse-stripped --stacktrace -q 'select' 2>&1 || :) if [[ $clickhouse_output =~ DB::LocalServer::main ]]; then echo "stripped_clickhouse_shows_symbols_names: [ OK ] 0 sec." else echo -e "stripped_clickhouse_shows_symbols_names: [ FAIL ] 0 sec. - clickhouse output:\n\n$clickhouse_output\n" fi ) | ts '%Y-%m-%d %H:%M:%S' | tee -a "$FASTTEST_OUTPUT/test_result.txt" if [ "$COPY_CLICKHOUSE_BINARY_TO_OUTPUT" -eq "1" ]; then mkdir -p "$FASTTEST_OUTPUT/binaries/" cp programs/clickhouse "$FASTTEST_OUTPUT/binaries/clickhouse" zstd --threads=0 programs/clickhouse-stripped -o "$FASTTEST_OUTPUT/binaries/clickhouse-stripped.zst" fi ccache_status ccache --evict-older-than 1d ||: ) } function configure { clickhouse-client --version clickhouse-test --help mkdir -p "$FASTTEST_DATA"{,/client-config} cp -a "$FASTTEST_SOURCE/programs/server/"{config,users}.xml "$FASTTEST_DATA" "$FASTTEST_SOURCE/tests/config/install.sh" "$FASTTEST_DATA" "$FASTTEST_DATA/client-config" cp -a "$FASTTEST_SOURCE/programs/server/config.d/log_to_console.xml" "$FASTTEST_DATA/config.d" # doesn't support SSL rm -f "$FASTTEST_DATA/config.d/secure_ports.xml" } function timeout_with_logging() { local exit_code=0 timeout -s TERM --preserve-status "${@}" || exit_code="${?}" if [[ "${exit_code}" -eq "124" ]] then echo "The command 'timeout ${*}' has been killed by timeout" fi return $exit_code } function run_tests { clickhouse-server --version clickhouse-test --help start_server set +e local NPROC NPROC=$(nproc) NPROC=$((NPROC / 2)) if [[ $NPROC == 0 ]]; then NPROC=1 fi export CLICKHOUSE_CONFIG_DIR=$FASTTEST_DATA export CLICKHOUSE_CONFIG="$FASTTEST_DATA/config.xml" export CLICKHOUSE_USER_FILES="$FASTTEST_DATA/user_files" export CLICKHOUSE_SCHEMA_FILES="$FASTTEST_DATA/format_schemas" local test_opts=( --hung-check --fast-tests-only --no-random-settings --no-random-merge-tree-settings --no-long --testname --shard --zookeeper --check-zookeeper-session --order random --print-time --report-logs-stats --jobs "${NPROC}" ) time clickhouse-test "${test_opts[@]}" -- "$FASTTEST_FOCUS" 2>&1 \ | ts '%Y-%m-%d %H:%M:%S' \ | tee -a "$FASTTEST_OUTPUT/test_result.txt" set -e clickhouse stop --pid-path "$FASTTEST_DATA" } export -f run_tests case "$stage" in "") ls -la ;& "clone_root") clone_root ;& "clone_submodules") clone_submodules 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/submodule_log.txt" ;& "run_cmake") cd "$FASTTEST_WORKSPACE" run_cmake ;& "build") build ;& "configure") # The `install_log.txt` is also needed for compatibility with old CI task -- # if there is no log, it will decide that build failed. configure 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/install_log.txt" ;& "run_tests") timeout_with_logging 35m bash -c run_tests ||: /process_functional_tests_result.py --in-results-dir "$FASTTEST_OUTPUT/" \ --out-results-file "$FASTTEST_OUTPUT/test_results.tsv" \ --out-status-file "$FASTTEST_OUTPUT/check_status.tsv" || echo -e "failure\tCannot parse results" > "$FASTTEST_OUTPUT/check_status.tsv" ;; *) echo "Unknown test stage '$stage'" exit 1 esac pstree -apgT jobs