ClickHouse/docker/test/stateful/run.sh

129 lines
5.9 KiB
Bash
Raw Normal View History

2020-07-08 08:41:39 +00:00
#!/bin/bash
set -e -x
dpkg -i package_folder/clickhouse-common-static_*.deb;
dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
dpkg -i package_folder/clickhouse-server_*.deb
dpkg -i package_folder/clickhouse-client_*.deb
dpkg -i package_folder/clickhouse-test_*.deb
2020-09-24 08:18:36 +00:00
# install test configs
/usr/share/clickhouse-test/config/install.sh
2020-07-08 08:41:39 +00:00
function start()
{
2021-03-13 10:22:48 +00:00
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
2021-03-16 16:39:31 +00:00
# NOTE We run "clickhouse server" instead of "clickhouse-server"
# to make "pidof clickhouse-server" return single pid of the main instance.
# We wil run main instance using "service clickhouse-server start"
2021-03-18 12:49:31 +00:00
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server1/config.xml --daemon \
-- --path /var/lib/clickhouse1/ --logger.stderr /var/log/clickhouse-server/stderr1.log \
--logger.log /var/log/clickhouse-server/clickhouse-server1.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server1.err.log \
2021-03-13 10:22:48 +00:00
--tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
2021-04-12 18:40:34 +00:00
--mysql_port 19004 --postgresql_port 19005 \
2021-03-29 20:04:50 +00:00
--keeper_server.tcp_port 19181 --keeper_server.server_id 2
2021-03-13 10:22:48 +00:00
2021-03-18 12:49:31 +00:00
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server2/config.xml --daemon \
-- --path /var/lib/clickhouse2/ --logger.stderr /var/log/clickhouse-server/stderr2.log \
--logger.log /var/log/clickhouse-server/clickhouse-server2.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server2.err.log \
2021-03-13 10:22:48 +00:00
--tcp_port 29000 --tcp_port_secure 29440 --http_port 28123 --https_port 28443 --interserver_http_port 29009 --tcp_with_proxy_port 29010 \
2021-04-12 18:40:34 +00:00
--mysql_port 29004 --postgresql_port 29005 \
2021-03-29 20:04:50 +00:00
--keeper_server.tcp_port 29181 --keeper_server.server_id 3
2021-03-13 10:22:48 +00:00
fi
counter=0
until clickhouse-client --query "SELECT 1"
do
if [ "$counter" -gt 120 ]
then
echo "Cannot start clickhouse-server"
cat /var/log/clickhouse-server/stdout.log
tail -n1000 /var/log/clickhouse-server/stderr.log
tail -n1000 /var/log/clickhouse-server/clickhouse-server.log
break
fi
timeout 120 service clickhouse-server start
sleep 0.5
2020-09-30 17:06:14 +00:00
counter=$((counter + 1))
done
}
start
2020-10-01 09:27:05 +00:00
# shellcheck disable=SC2086 # No quotes because I want to split it into words.
/s3downloader --dataset-names $DATASETS
2020-07-08 08:41:39 +00:00
chmod 777 -R /var/lib/clickhouse
clickhouse-client --query "SHOW DATABASES"
2021-01-05 00:02:40 +00:00
2021-03-16 16:39:31 +00:00
clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary"
2021-01-05 00:02:40 +00:00
service clickhouse-server restart
# Wait for server to start accepting connections
for _ in {1..120}; do
clickhouse-client --query "SELECT 1" && break
sleep 1
done
2020-07-08 08:41:39 +00:00
clickhouse-client --query "SHOW TABLES FROM datasets"
2021-03-16 16:39:31 +00:00
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
clickhouse-client --query "CREATE DATABASE test ON CLUSTER 'test_cluster_database_replicated'
ENGINE=Replicated('/test/clickhouse/db/test', '{shard}', '{replica}')"
2020-07-08 08:41:39 +00:00
2021-03-16 16:39:31 +00:00
clickhouse-client --query "CREATE TABLE test.hits AS datasets.hits_v1"
clickhouse-client --query "CREATE TABLE test.visits AS datasets.visits_v1"
2021-03-16 16:39:31 +00:00
clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1"
clickhouse-client --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1"
clickhouse-client --query "DROP TABLE datasets.hits_v1"
clickhouse-client --query "DROP TABLE datasets.visits_v1"
MAX_RUN_TIME=$((MAX_RUN_TIME < 9000 ? MAX_RUN_TIME : 9000)) # min(MAX_RUN_TIME, 2.5 hours)
MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000)) # set to 2.5 hours if 0 (unlimited)
else
clickhouse-client --query "CREATE DATABASE test"
clickhouse-client --query "SHOW TABLES FROM test"
clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
2021-02-15 10:26:34 +00:00
fi
2021-03-16 16:39:31 +00:00
clickhouse-client --query "SHOW TABLES FROM test"
clickhouse-client --query "SELECT count() FROM test.hits"
clickhouse-client --query "SELECT count() FROM test.visits"
function run_tests()
{
set -x
# We can have several additional options so we path them as array because it's
# more idiologically correct.
read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}"
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
ADDITIONAL_OPTIONS+=('--replicated-database')
fi
2021-03-16 20:01:20 +00:00
clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --use-skip-list --print-time "${ADDITIONAL_OPTIONS[@]}" \
2021-03-16 16:39:31 +00:00
"$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
}
export -f run_tests
timeout "$MAX_RUN_TIME" bash -c run_tests ||:
2021-02-20 20:04:24 +00:00
2021-02-22 13:53:43 +00:00
./process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
2021-02-20 20:04:24 +00:00
2021-06-15 20:52:29 +00:00
grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||:
2021-02-22 13:53:43 +00:00
pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz ||:
mv /var/log/clickhouse-server/stderr.log /test_output/ ||:
if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
tar -chf /test_output/clickhouse_coverage.tar.gz /profraw ||:
fi
2021-03-13 10:22:48 +00:00
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
2021-06-15 20:52:29 +00:00
grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server1.log ||:
grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server2.log ||:
2021-03-18 12:49:31 +00:00
pigz < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.gz ||:
2021-03-13 10:22:48 +00:00
pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||:
2021-03-18 12:49:31 +00:00
mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:
2021-03-13 10:22:48 +00:00
mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
fi