mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
154 lines
6.8 KiB
Bash
Executable File
154 lines
6.8 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -e -x
|
|
|
|
# Choose random timezone for this test run
|
|
TZ="$(grep -v '#' /usr/share/zoneinfo/zone.tab | awk '{print $3}' | shuf | head -n1)"
|
|
echo "Choosen random timezone $TZ"
|
|
ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" > /etc/timezone
|
|
|
|
dpkg -i package_folder/clickhouse-common-static_*.deb;
|
|
dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
|
|
dpkg -i package_folder/clickhouse-server_*.deb
|
|
dpkg -i package_folder/clickhouse-client_*.deb
|
|
dpkg -i package_folder/clickhouse-test_*.deb
|
|
|
|
# install test configs
|
|
/usr/share/clickhouse-test/config/install.sh
|
|
|
|
function start()
|
|
{
|
|
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
|
# NOTE We run "clickhouse server" instead of "clickhouse-server"
|
|
# to make "pidof clickhouse-server" return single pid of the main instance.
|
|
# We wil run main instance using "service clickhouse-server start"
|
|
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server1/config.xml --daemon \
|
|
-- --path /var/lib/clickhouse1/ --logger.stderr /var/log/clickhouse-server/stderr1.log \
|
|
--logger.log /var/log/clickhouse-server/clickhouse-server1.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server1.err.log \
|
|
--tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
|
|
--mysql_port 19004 --postgresql_port 19005 \
|
|
--keeper_server.tcp_port 19181 --keeper_server.server_id 2
|
|
|
|
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server2/config.xml --daemon \
|
|
-- --path /var/lib/clickhouse2/ --logger.stderr /var/log/clickhouse-server/stderr2.log \
|
|
--logger.log /var/log/clickhouse-server/clickhouse-server2.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server2.err.log \
|
|
--tcp_port 29000 --tcp_port_secure 29440 --http_port 28123 --https_port 28443 --interserver_http_port 29009 --tcp_with_proxy_port 29010 \
|
|
--mysql_port 29004 --postgresql_port 29005 \
|
|
--keeper_server.tcp_port 29181 --keeper_server.server_id 3
|
|
fi
|
|
|
|
counter=0
|
|
until clickhouse-client --query "SELECT 1"
|
|
do
|
|
if [ "$counter" -gt 120 ]
|
|
then
|
|
echo "Cannot start clickhouse-server"
|
|
cat /var/log/clickhouse-server/stdout.log
|
|
tail -n1000 /var/log/clickhouse-server/stderr.log
|
|
tail -n1000 /var/log/clickhouse-server/clickhouse-server.log
|
|
break
|
|
fi
|
|
timeout 120 service clickhouse-server start
|
|
sleep 0.5
|
|
counter=$((counter + 1))
|
|
done
|
|
}
|
|
|
|
start
|
|
# shellcheck disable=SC2086 # No quotes because I want to split it into words.
|
|
/s3downloader --url-prefix "$S3_URL" --dataset-names $DATASETS
|
|
chmod 777 -R /var/lib/clickhouse
|
|
clickhouse-client --query "SHOW DATABASES"
|
|
|
|
clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary"
|
|
|
|
service clickhouse-server restart
|
|
|
|
# Wait for server to start accepting connections
|
|
for _ in {1..120}; do
|
|
clickhouse-client --query "SELECT 1" && break
|
|
sleep 1
|
|
done
|
|
|
|
clickhouse-client --query "SHOW TABLES FROM datasets"
|
|
|
|
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
|
clickhouse-client --query "CREATE DATABASE test ON CLUSTER 'test_cluster_database_replicated'
|
|
ENGINE=Replicated('/test/clickhouse/db/test', '{shard}', '{replica}')"
|
|
|
|
clickhouse-client --query "CREATE TABLE test.hits AS datasets.hits_v1"
|
|
clickhouse-client --query "CREATE TABLE test.visits AS datasets.visits_v1"
|
|
|
|
clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1"
|
|
clickhouse-client --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1"
|
|
|
|
clickhouse-client --query "DROP TABLE datasets.hits_v1"
|
|
clickhouse-client --query "DROP TABLE datasets.visits_v1"
|
|
|
|
MAX_RUN_TIME=$((MAX_RUN_TIME < 9000 ? MAX_RUN_TIME : 9000)) # min(MAX_RUN_TIME, 2.5 hours)
|
|
MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000)) # set to 2.5 hours if 0 (unlimited)
|
|
else
|
|
clickhouse-client --query "CREATE DATABASE test"
|
|
clickhouse-client --query "SHOW TABLES FROM test"
|
|
clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
|
|
clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
|
|
fi
|
|
|
|
clickhouse-client --query "SHOW TABLES FROM test"
|
|
clickhouse-client --query "SELECT count() FROM test.hits"
|
|
clickhouse-client --query "SELECT count() FROM test.visits"
|
|
|
|
function run_tests()
|
|
{
|
|
set -x
|
|
# We can have several additional options so we path them as array because it's
|
|
# more idiologically correct.
|
|
read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}"
|
|
|
|
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
|
ADDITIONAL_OPTIONS+=('--replicated-database')
|
|
fi
|
|
|
|
set +e
|
|
clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --no-stateless --hung-check --print-time \
|
|
--skip 00168_parallel_processing_on_replicas "${ADDITIONAL_OPTIONS[@]}" \
|
|
"$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
|
|
|
clickhouse-test --timeout 1200 --testname --shard --zookeeper --check-zookeeper-session --no-stateless --hung-check --print-time \
|
|
00168_parallel_processing_on_replicas "${ADDITIONAL_OPTIONS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a test_output/test_result.txt
|
|
|
|
set -e
|
|
}
|
|
|
|
export -f run_tests
|
|
timeout "$MAX_RUN_TIME" bash -c run_tests ||:
|
|
|
|
echo "Files in current directory"
|
|
ls -la ./
|
|
echo "Files in root directory"
|
|
ls -la /
|
|
|
|
/process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
|
|
|
|
grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||:
|
|
|
|
pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz ||:
|
|
# FIXME: remove once only github actions will be left
|
|
rm /var/log/clickhouse-server/clickhouse-server.log
|
|
mv /var/log/clickhouse-server/stderr.log /test_output/ ||:
|
|
|
|
if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
|
|
tar -chf /test_output/clickhouse_coverage.tar.gz /profraw ||:
|
|
fi
|
|
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
|
grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server1.log ||:
|
|
grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server2.log ||:
|
|
pigz < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.gz ||:
|
|
pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||:
|
|
# FIXME: remove once only github actions will be left
|
|
rm /var/log/clickhouse-server/clickhouse-server1.log
|
|
rm /var/log/clickhouse-server/clickhouse-server2.log
|
|
mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:
|
|
mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
|
|
fi
|