ClickHouse/utils/durability-test/durability-test.sh

166 lines
3.7 KiB
Bash
Raw Normal View History

2020-09-01 01:46:40 +00:00
#!/bin/bash
2020-09-10 23:24:16 +00:00
: '
A simple test for durability. It starts up clickhouse server in qemu VM and runs
inserts via clickhouse benchmark tool. Then it kills VM in random moment and
checks whether table contains broken parts. With enabled fsync no broken parts
should be appeared.
Usage:
./install.sh
./durability-test.sh <table name> <file with create query> <file with insert query>
'
2020-09-01 01:46:40 +00:00
URL=http://cloud-images.ubuntu.com/bionic/current
IMAGE=bionic-server-cloudimg-amd64.img
SSH_PORT=11022
PASSWORD=root
TABLE_NAME=$1
CREATE_QUERY=$2
INSERT_QUERY=$3
if [[ -z $TABLE_NAME || -z $CREATE_QUERY || -z $INSERT_QUERY ]]; then
echo "Required 3 arguments: table name, file with create query, file with insert query"
exit 1
fi
function run()
{
2020-09-01 15:26:49 +00:00
sshpass -p $PASSWORD ssh -p $SSH_PORT root@localhost "$1" 2>/dev/null
2020-09-01 01:46:40 +00:00
}
function copy()
{
sshpass -p $PASSWORD scp -r -P $SSH_PORT "$1" "root@localhost:$2" 2>/dev/null
2020-09-01 01:46:40 +00:00
}
function wait_vm_for_start()
{
echo "Waiting until VM started..."
started=0
for _ in {0..100}; do
if run "exit"; then
2020-09-01 01:46:40 +00:00
started=1
break
fi
2020-09-01 01:46:40 +00:00
sleep 1s
done
if ((started == 0)); then
echo "Can't start or connect to VM."
exit 1
fi
echo "Started VM"
}
function wait_clickhouse_for_start()
{
echo "Waiting until ClickHouse started..."
started=0
for _ in {0..30}; do
if run "clickhouse client --query 'select 1'" > /dev/null; then
2020-09-01 01:46:40 +00:00
started=1
break
fi
sleep 1s
done
if ((started == 0)); then
echo "Can't start ClickHouse."
fi
echo "Started ClickHouse"
}
echo "Downloading image"
curl -O $URL/$IMAGE
qemu-img resize $IMAGE +10G
2020-09-01 15:26:49 +00:00
virt-customize -a $IMAGE --root-password password:$PASSWORD > /dev/null 2>&1
2020-09-01 01:46:40 +00:00
virt-copy-in -a $IMAGE sshd_config /etc/ssh
echo "Starting VM"
chmod +x ./startup.exp
./startup.exp > qemu.log 2>&1 &
wait_vm_for_start
echo "Preparing VM"
# Resize partition
run "growpart /dev/sda 1 && resize2fs /dev/sda1"
if [[ -z $CLICKHOUSE_BINARY ]]; then
CLICKHOUSE_BINARY=/usr/bin/clickhouse
fi
if [[ -z $CLICKHOUSE_CONFIG_DIR ]]; then
CLICKHOUSE_CONFIG_DIR=/etc/clickhouse-server
fi
echo "Using ClickHouse binary: $CLICKHOUSE_BINARY"
echo "Using ClickHouse config from: $CLICKHOUSE_CONFIG_DIR"
2020-09-01 01:46:40 +00:00
copy "$CLICKHOUSE_BINARY" /usr/bin
copy "$CLICKHOUSE_CONFIG_DIR" /etc
2020-09-01 01:46:40 +00:00
run "mv /etc/$CLICKHOUSE_CONFIG_DIR /etc/clickhouse-server"
echo "Prepared VM"
echo "Starting ClickHouse"
run "clickhouse server --config-file=/etc/clickhouse-server/config.xml > clickhouse-server.log 2>&1" &
wait_clickhouse_for_start
query=$(cat "$CREATE_QUERY")
echo "Executing query: $query"
2020-09-01 01:46:40 +00:00
run "clickhouse client --query '$query'"
query=$(cat "$INSERT_QUERY")
echo "Will run in a loop query: $query"
2020-09-01 15:26:49 +00:00
run "clickhouse benchmark <<< '$query' -c 8" &
2020-09-01 01:46:40 +00:00
echo "Running queries"
pid=$(pidof qemu-system-x86_64)
2020-09-01 15:26:49 +00:00
sec=$(( (RANDOM % 5) + 25 ))
2020-09-01 01:46:40 +00:00
ms=$(( RANDOM % 1000 ))
echo "Will kill VM in $sec.$ms sec"
sleep $sec.$ms
kill -9 "$pid"
2020-09-01 01:46:40 +00:00
echo "Restarting"
2020-09-01 15:26:49 +00:00
sleep 5s
2020-09-01 01:46:40 +00:00
./startup.exp > qemu.log 2>&1 &
wait_vm_for_start
run "rm -r *data/system"
run "clickhouse server --config-file=/etc/clickhouse-server/config.xml > clickhouse-server.log 2>&1" &
wait_clickhouse_for_start
pid=$(pidof qemu-system-x86_64)
result=$(run "grep $TABLE_NAME clickhouse-server.log | grep 'Caught exception while loading metadata'")
2020-09-01 01:46:40 +00:00
if [[ -n $result ]]; then
echo "FAIL. Can't attach table:"
echo "$result"
kill -9 "$pid"
2020-09-01 01:46:40 +00:00
exit 1
fi
result=$(run "grep $TABLE_NAME clickhouse-server.log | grep 'Considering to remove broken part'")
2020-09-01 01:46:40 +00:00
if [[ -n $result ]]; then
echo "FAIL. Have broken parts:"
echo "$result"
kill -9 "$pid"
2020-09-01 01:46:40 +00:00
exit 1
fi
kill -9 "$pid"
2020-09-01 01:46:40 +00:00
echo OK