2020-07-08 08:41:39 +00:00
|
|
|
#!/bin/bash
|
|
|
|
|
2020-10-21 20:11:35 +00:00
|
|
|
# fail on errors, verbose and export all env variables
|
|
|
|
set -e -x -a
|
2020-07-08 08:41:39 +00:00
|
|
|
|
2020-09-28 12:11:36 +00:00
|
|
|
# Choose random timezone for this test run.
|
2023-01-01 19:53:06 +00:00
|
|
|
TZ="$(rg -v '#' /usr/share/zoneinfo/zone.tab | awk '{print $3}' | shuf | head -n1)"
|
2020-09-28 12:11:36 +00:00
|
|
|
echo "Choosen random timezone $TZ"
|
2021-01-06 04:20:33 +00:00
|
|
|
ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" > /etc/timezone
|
2020-09-28 12:11:36 +00:00
|
|
|
|
2020-07-08 08:41:39 +00:00
|
|
|
dpkg -i package_folder/clickhouse-common-static_*.deb
|
|
|
|
dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
|
|
|
|
dpkg -i package_folder/clickhouse-server_*.deb
|
|
|
|
dpkg -i package_folder/clickhouse-client_*.deb
|
2022-02-15 12:03:51 +00:00
|
|
|
|
|
|
|
ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
|
2020-07-08 08:41:39 +00:00
|
|
|
|
2023-06-06 10:23:00 +00:00
|
|
|
# shellcheck disable=SC1091
|
2023-06-13 15:53:19 +00:00
|
|
|
source /usr/share/clickhouse-test/ci/attach_gdb.lib || true # FIXME: to not break old builds, clean on 2023-09-01
|
2023-06-05 18:21:40 +00:00
|
|
|
|
2023-06-30 14:41:27 +00:00
|
|
|
# shellcheck disable=SC1091
|
2023-07-02 09:52:50 +00:00
|
|
|
source /usr/share/clickhouse-test/ci/utils.lib || true # FIXME: to not break old builds, clean on 2023-09-01
|
2023-06-30 14:41:27 +00:00
|
|
|
|
2020-09-24 08:18:36 +00:00
|
|
|
# install test configs
|
|
|
|
/usr/share/clickhouse-test/config/install.sh
|
2020-07-08 08:41:39 +00:00
|
|
|
|
2022-08-15 22:58:09 +00:00
|
|
|
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
|
|
|
echo "Azure is disabled"
|
|
|
|
else
|
|
|
|
azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
|
|
|
|
fi
|
2022-08-05 12:00:45 +00:00
|
|
|
|
2022-06-10 12:51:18 +00:00
|
|
|
./setup_minio.sh stateless
|
2022-06-03 13:54:29 +00:00
|
|
|
./setup_hdfs_minicluster.sh
|
2021-08-26 13:12:42 +00:00
|
|
|
|
2020-12-21 10:06:36 +00:00
|
|
|
# For flaky check we also enable thread fuzzer
|
|
|
|
if [ "$NUM_TRIES" -gt "1" ]; then
|
|
|
|
export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
|
|
|
|
export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
|
|
|
|
export THREAD_FUZZER_SLEEP_TIME_US=100000
|
2020-12-21 20:24:16 +00:00
|
|
|
|
|
|
|
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
|
|
|
|
export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
|
|
|
|
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_MIGRATE_PROBABILITY=1
|
|
|
|
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_MIGRATE_PROBABILITY=1
|
|
|
|
|
|
|
|
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001
|
|
|
|
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
|
|
|
|
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
|
|
|
|
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
|
|
|
|
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
|
|
|
|
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
|
|
|
|
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
|
|
|
|
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
|
|
|
|
|
2022-05-26 10:03:12 +00:00
|
|
|
mkdir -p /var/run/clickhouse-server
|
2020-12-21 10:06:36 +00:00
|
|
|
# simpliest way to forward env variables to server
|
2022-05-26 10:03:12 +00:00
|
|
|
sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml --daemon --pid-file /var/run/clickhouse-server/clickhouse-server.pid
|
2020-12-21 10:06:36 +00:00
|
|
|
else
|
2021-07-03 23:13:32 +00:00
|
|
|
sudo clickhouse start
|
2020-12-21 10:06:36 +00:00
|
|
|
fi
|
2020-07-08 08:41:39 +00:00
|
|
|
|
2021-03-13 01:25:13 +00:00
|
|
|
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
2022-05-26 10:03:12 +00:00
|
|
|
mkdir -p /var/run/clickhouse-server1
|
|
|
|
sudo chown clickhouse:clickhouse /var/run/clickhouse-server1
|
2021-03-18 12:49:31 +00:00
|
|
|
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server1/config.xml --daemon \
|
2022-05-26 10:03:12 +00:00
|
|
|
--pid-file /var/run/clickhouse-server1/clickhouse-server.pid \
|
2021-03-18 12:49:31 +00:00
|
|
|
-- --path /var/lib/clickhouse1/ --logger.stderr /var/log/clickhouse-server/stderr1.log \
|
|
|
|
--logger.log /var/log/clickhouse-server/clickhouse-server1.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server1.err.log \
|
2021-03-13 01:25:13 +00:00
|
|
|
--tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
|
2021-04-12 18:40:34 +00:00
|
|
|
--mysql_port 19004 --postgresql_port 19005 \
|
2021-03-29 20:04:50 +00:00
|
|
|
--keeper_server.tcp_port 19181 --keeper_server.server_id 2 \
|
2022-08-06 10:00:53 +00:00
|
|
|
--prometheus.port 19988 \
|
2021-03-13 01:25:13 +00:00
|
|
|
--macros.replica r2 # It doesn't work :(
|
2021-03-13 10:22:48 +00:00
|
|
|
|
2022-05-26 10:03:12 +00:00
|
|
|
mkdir -p /var/run/clickhouse-server2
|
|
|
|
sudo chown clickhouse:clickhouse /var/run/clickhouse-server2
|
2021-03-18 12:49:31 +00:00
|
|
|
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server2/config.xml --daemon \
|
2022-05-26 10:03:12 +00:00
|
|
|
--pid-file /var/run/clickhouse-server2/clickhouse-server.pid \
|
2021-03-18 12:49:31 +00:00
|
|
|
-- --path /var/lib/clickhouse2/ --logger.stderr /var/log/clickhouse-server/stderr2.log \
|
|
|
|
--logger.log /var/log/clickhouse-server/clickhouse-server2.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server2.err.log \
|
2021-03-13 10:22:48 +00:00
|
|
|
--tcp_port 29000 --tcp_port_secure 29440 --http_port 28123 --https_port 28443 --interserver_http_port 29009 --tcp_with_proxy_port 29010 \
|
2021-04-12 18:40:34 +00:00
|
|
|
--mysql_port 29004 --postgresql_port 29005 \
|
2021-03-29 20:04:50 +00:00
|
|
|
--keeper_server.tcp_port 29181 --keeper_server.server_id 3 \
|
2022-08-06 10:00:53 +00:00
|
|
|
--prometheus.port 29988 \
|
2021-03-13 10:22:48 +00:00
|
|
|
--macros.shard s2 # It doesn't work :(
|
2021-03-16 16:39:31 +00:00
|
|
|
|
|
|
|
MAX_RUN_TIME=$((MAX_RUN_TIME < 9000 ? MAX_RUN_TIME : 9000)) # min(MAX_RUN_TIME, 2.5 hours)
|
|
|
|
MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000)) # set to 2.5 hours if 0 (unlimited)
|
2020-07-08 08:41:39 +00:00
|
|
|
fi
|
|
|
|
|
2021-03-13 01:25:13 +00:00
|
|
|
sleep 5
|
|
|
|
|
2023-06-13 15:53:19 +00:00
|
|
|
attach_gdb_to_clickhouse || true # FIXME: to not break old builds, clean on 2023-09-01
|
2023-06-02 15:00:24 +00:00
|
|
|
|
2023-07-02 09:52:50 +00:00
|
|
|
function fn_exists() {
|
|
|
|
declare -F "$1" > /dev/null;
|
|
|
|
}
|
|
|
|
|
|
|
|
# FIXME: to not break old builds, clean on 2023-09-01
|
|
|
|
function try_run_with_retry() {
|
|
|
|
local total_retries="$1"
|
|
|
|
shift
|
|
|
|
|
|
|
|
if fn_exists run_with_retry; then
|
|
|
|
run_with_retry "$total_retries" "$@"
|
|
|
|
else
|
|
|
|
"$@"
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2020-09-29 09:44:28 +00:00
|
|
|
function run_tests()
|
|
|
|
{
|
2021-03-16 16:39:31 +00:00
|
|
|
set -x
|
2022-08-25 23:59:14 +00:00
|
|
|
# We can have several additional options so we pass them as array because it is more ideologically correct.
|
2020-10-22 14:33:23 +00:00
|
|
|
read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}"
|
2020-10-23 10:18:06 +00:00
|
|
|
|
2022-08-25 23:59:14 +00:00
|
|
|
HIGH_LEVEL_COVERAGE=YES
|
|
|
|
|
2022-08-11 19:27:26 +00:00
|
|
|
# Use random order in flaky check
|
2020-10-23 10:18:06 +00:00
|
|
|
if [ "$NUM_TRIES" -gt "1" ]; then
|
2021-03-29 18:24:29 +00:00
|
|
|
ADDITIONAL_OPTIONS+=('--order=random')
|
2022-08-25 23:59:14 +00:00
|
|
|
HIGH_LEVEL_COVERAGE=NO
|
2020-10-23 10:18:06 +00:00
|
|
|
fi
|
|
|
|
|
2022-02-04 11:32:11 +00:00
|
|
|
if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
|
|
|
|
ADDITIONAL_OPTIONS+=('--s3-storage')
|
|
|
|
fi
|
|
|
|
|
2021-02-15 10:26:34 +00:00
|
|
|
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
|
|
|
ADDITIONAL_OPTIONS+=('--replicated-database')
|
2021-06-11 12:28:27 +00:00
|
|
|
ADDITIONAL_OPTIONS+=('--jobs')
|
|
|
|
ADDITIONAL_OPTIONS+=('2')
|
2021-04-01 17:57:31 +00:00
|
|
|
else
|
|
|
|
# Too many tests fail for DatabaseReplicated in parallel. All other
|
|
|
|
# configurations are OK.
|
|
|
|
ADDITIONAL_OPTIONS+=('--jobs')
|
|
|
|
ADDITIONAL_OPTIONS+=('8')
|
2021-02-15 10:26:34 +00:00
|
|
|
fi
|
|
|
|
|
2021-12-10 15:39:02 +00:00
|
|
|
if [[ -n "$RUN_BY_HASH_NUM" ]] && [[ -n "$RUN_BY_HASH_TOTAL" ]]; then
|
|
|
|
ADDITIONAL_OPTIONS+=('--run-by-hash-num')
|
|
|
|
ADDITIONAL_OPTIONS+=("$RUN_BY_HASH_NUM")
|
|
|
|
ADDITIONAL_OPTIONS+=('--run-by-hash-total')
|
|
|
|
ADDITIONAL_OPTIONS+=("$RUN_BY_HASH_TOTAL")
|
2022-08-25 23:59:14 +00:00
|
|
|
HIGH_LEVEL_COVERAGE=NO
|
2021-12-10 15:39:02 +00:00
|
|
|
fi
|
|
|
|
|
2022-06-23 07:59:13 +00:00
|
|
|
if [[ -n "$USE_DATABASE_ORDINARY" ]] && [[ "$USE_DATABASE_ORDINARY" -eq 1 ]]; then
|
|
|
|
ADDITIONAL_OPTIONS+=('--db-engine=Ordinary')
|
|
|
|
fi
|
|
|
|
|
2022-08-25 23:59:14 +00:00
|
|
|
if [[ "${HIGH_LEVEL_COVERAGE}" = "YES" ]]; then
|
|
|
|
ADDITIONAL_OPTIONS+=('--report-coverage')
|
|
|
|
fi
|
|
|
|
|
2023-01-17 19:04:25 +00:00
|
|
|
ADDITIONAL_OPTIONS+=('--report-logs-stats')
|
|
|
|
|
2023-07-02 09:52:50 +00:00
|
|
|
try_run_with_retry 10 clickhouse-client -q "insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '/test/chroot/', '')"
|
2023-05-25 09:50:14 +00:00
|
|
|
|
2021-10-06 20:07:30 +00:00
|
|
|
set +e
|
2021-10-29 11:45:09 +00:00
|
|
|
clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
|
2023-02-03 13:34:18 +00:00
|
|
|
--test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
|
|
|
|
| ts '%Y-%m-%d %H:%M:%S' \
|
|
|
|
| tee -a test_output/test_result.txt
|
2021-10-06 20:07:30 +00:00
|
|
|
set -e
|
2020-09-29 09:44:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
export -f run_tests
|
|
|
|
|
2022-08-11 19:27:26 +00:00
|
|
|
if [ "$NUM_TRIES" -gt "1" ]; then
|
|
|
|
# We don't run tests with Ordinary database in PRs, only in master.
|
|
|
|
# So run new/changed tests with Ordinary at least once in flaky check.
|
|
|
|
timeout "$MAX_RUN_TIME" bash -c 'NUM_TRIES=1; USE_DATABASE_ORDINARY=1; run_tests' \
|
|
|
|
| sed 's/All tests have finished//' | sed 's/No tests were run//' ||:
|
|
|
|
fi
|
|
|
|
|
2020-12-28 11:46:53 +00:00
|
|
|
timeout "$MAX_RUN_TIME" bash -c run_tests ||:
|
2021-01-14 10:42:51 +00:00
|
|
|
|
2021-12-10 16:38:20 +00:00
|
|
|
echo "Files in current directory"
|
|
|
|
ls -la ./
|
|
|
|
echo "Files in root directory"
|
|
|
|
ls -la /
|
|
|
|
|
|
|
|
/process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
|
2021-02-20 20:04:24 +00:00
|
|
|
|
2021-03-30 10:49:43 +00:00
|
|
|
clickhouse-client -q "system flush logs" ||:
|
2021-03-29 11:47:57 +00:00
|
|
|
|
2022-05-03 18:10:07 +00:00
|
|
|
# Stop server so we can safely read data with clickhouse-local.
|
|
|
|
# Why do we read data with clickhouse-local?
|
|
|
|
# Because it's the simplest way to read it when server has crashed.
|
2022-05-26 10:03:12 +00:00
|
|
|
sudo clickhouse stop ||:
|
2022-05-03 22:10:40 +00:00
|
|
|
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
2022-05-26 10:03:12 +00:00
|
|
|
sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||:
|
|
|
|
sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||:
|
2022-05-03 22:10:40 +00:00
|
|
|
fi
|
|
|
|
|
2023-02-14 12:12:53 +00:00
|
|
|
rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server.log ||:
|
2023-03-23 14:36:17 +00:00
|
|
|
rg -A50 -Fa "============" /var/log/clickhouse-server/stderr.log ||:
|
2023-01-01 20:17:43 +00:00
|
|
|
zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst &
|
2022-03-30 17:49:39 +00:00
|
|
|
|
|
|
|
# Compress tables.
|
|
|
|
#
|
|
|
|
# NOTE:
|
|
|
|
# - that due to tests with s3 storage we cannot use /var/lib/clickhouse/data
|
|
|
|
# directly
|
|
|
|
# - even though ci auto-compress some files (but not *.tsv) it does this only
|
|
|
|
# for files >64MB, we want this files to be compressed explicitly
|
2022-03-31 09:35:04 +00:00
|
|
|
for table in query_log zookeeper_log trace_log transactions_info_log
|
2022-03-30 17:49:39 +00:00
|
|
|
do
|
2023-01-01 20:17:43 +00:00
|
|
|
clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst ||:
|
2022-03-30 17:49:39 +00:00
|
|
|
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
2023-01-01 20:17:43 +00:00
|
|
|
clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst ||:
|
|
|
|
clickhouse-local --path /var/lib/clickhouse2/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst ||:
|
2022-03-30 17:49:39 +00:00
|
|
|
fi
|
|
|
|
done
|
2021-04-23 18:50:02 +00:00
|
|
|
|
|
|
|
# Also export trace log in flamegraph-friendly format.
|
|
|
|
for trace_type in CPU Memory Real
|
|
|
|
do
|
2022-09-19 14:10:00 +00:00
|
|
|
clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "
|
2021-04-23 18:50:02 +00:00
|
|
|
select
|
|
|
|
arrayStringConcat((arrayMap(x -> concat(splitByChar('/', addressToLine(x))[-1], '#', demangle(addressToSymbol(x)) ), trace)), ';') AS stack,
|
|
|
|
count(*) AS samples
|
|
|
|
from system.trace_log
|
2021-04-23 20:03:53 +00:00
|
|
|
where trace_type = '$trace_type'
|
2021-04-23 18:50:02 +00:00
|
|
|
group by trace
|
|
|
|
order by samples desc
|
|
|
|
settings allow_introspection_functions = 1
|
|
|
|
format TabSeparated" \
|
2023-01-01 20:17:43 +00:00
|
|
|
| zstd --threads=0 > "/test_output/trace-log-$trace_type-flamegraph.tsv.zst" ||:
|
2021-04-23 18:50:02 +00:00
|
|
|
done
|
|
|
|
|
2021-03-29 11:47:57 +00:00
|
|
|
|
2021-11-19 18:17:47 +00:00
|
|
|
# Compressed (FIXME: remove once only github actions will be left)
|
|
|
|
rm /var/log/clickhouse-server/clickhouse-server.log
|
2021-02-22 13:53:43 +00:00
|
|
|
mv /var/log/clickhouse-server/stderr.log /test_output/ ||:
|
|
|
|
if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
|
2023-01-01 20:17:43 +00:00
|
|
|
tar --zstd -chf /test_output/clickhouse_coverage.tar.zst /profraw ||:
|
2021-02-22 13:53:43 +00:00
|
|
|
fi
|
2021-10-23 16:58:10 +00:00
|
|
|
|
2021-04-14 15:04:57 +00:00
|
|
|
tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
|
2021-03-13 01:25:13 +00:00
|
|
|
|
|
|
|
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
2023-02-14 12:12:53 +00:00
|
|
|
rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server1.log ||:
|
|
|
|
rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server2.log ||:
|
2023-01-01 20:17:43 +00:00
|
|
|
zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.zst ||:
|
|
|
|
zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.zst ||:
|
2021-11-19 18:17:47 +00:00
|
|
|
# FIXME: remove once only github actions will be left
|
|
|
|
rm /var/log/clickhouse-server/clickhouse-server1.log
|
|
|
|
rm /var/log/clickhouse-server/clickhouse-server2.log
|
2021-03-18 12:49:31 +00:00
|
|
|
mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:
|
2021-03-13 01:25:13 +00:00
|
|
|
mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
|
2021-04-14 15:04:57 +00:00
|
|
|
tar -chf /test_output/coordination1.tar /var/lib/clickhouse1/coordination ||:
|
|
|
|
tar -chf /test_output/coordination2.tar /var/lib/clickhouse2/coordination ||:
|
2021-03-13 01:25:13 +00:00
|
|
|
fi
|