ClickHouse/utils/durability-test/durability-test.sh

#!/bin/bash

: '
A simple test for durability. It starts up clickhouse server in qemu VM and runs
inserts via clickhouse benchmark tool. Then it kills VM in random moment and
checks whether table contains broken parts. With enabled fsync no broken parts
should be appeared.

Usage:

./install.sh
./durability-test.sh <table name> <file with create query> <file with insert query>
'

URL=http://cloud-images.ubuntu.com/bionic/current
IMAGE=bionic-server-cloudimg-amd64.img
SSH_PORT=11022
CLICKHOUSE_PORT=9090
PASSWORD=root

TABLE_NAME=$1
CREATE_QUERY=$2
INSERT_QUERY=$3

if [[ -z $TABLE_NAME || -z $CREATE_QUERY || -z $INSERT_QUERY ]]; then
    echo "Required 3 arguments: table name, file with create query, file with insert query"
    exit 1
fi

function run()
{
    sshpass -p $PASSWORD ssh -p $SSH_PORT root@localhost "$1" 2>/dev/null
}

function copy()
{
    sshpass -p $PASSWORD scp -r -P $SSH_PORT $1 root@localhost:$2 2>/dev/null
}

function wait_vm_for_start()
{
    echo "Waiting until VM started..."
    started=0
    for i in {0..100}; do
        run "exit"
        if [ $? -eq 0 ]; then
            started=1
            break
        fi 
        sleep 1s
    done

    if ((started == 0)); then
        echo "Can't start or connect to VM."
        exit 1
    fi

    echo "Started VM"
}

function wait_clickhouse_for_start()
{
    echo "Waiting until ClickHouse started..."
    started=0
    for i in {0..30}; do
        run "clickhouse client --query 'select 1'" > /dev/null
        if [ $? -eq 0 ]; then
            started=1
            break
        fi
        sleep 1s
    done

    if ((started == 0)); then
        echo "Can't start ClickHouse."
    fi

    echo "Started ClickHouse"
}

echo "Downloading image"
curl -O $URL/$IMAGE

qemu-img resize $IMAGE +10G
virt-customize -a $IMAGE --root-password password:$PASSWORD > /dev/null 2>&1
virt-copy-in -a $IMAGE sshd_config /etc/ssh

echo "Starting VM"

chmod +x ./startup.exp
./startup.exp > qemu.log 2>&1 &

wait_vm_for_start

echo "Preparing VM"

# Resize partition
run "growpart /dev/sda 1 && resize2fs /dev/sda1"

if [[ -z $CLICKHOUSE_BINARY ]]; then
    CLICKHOUSE_BINARY=/usr/bin/clickhouse
fi

if [[ -z $CLICKHOUSE_CONFIG_DIR ]]; then
    CLICKHOUSE_CONFIG_DIR=/etc/clickhouse-server
fi

echo "Using ClickHouse binary:" $CLICKHOUSE_BINARY
echo "Using ClickHouse config from:" $CLICKHOUSE_CONFIG_DIR

copy $CLICKHOUSE_BINARY /usr/bin
copy $CLICKHOUSE_CONFIG_DIR /etc
run "mv /etc/$CLICKHOUSE_CONFIG_DIR /etc/clickhouse-server"

echo "Prepared VM"
echo "Starting ClickHouse"

run "clickhouse server --config-file=/etc/clickhouse-server/config.xml > clickhouse-server.log 2>&1" &
wait_clickhouse_for_start

query=`cat $CREATE_QUERY`
echo "Executing query:" $query
run "clickhouse client --query '$query'"

query=`cat $INSERT_QUERY`
echo "Will run in a loop query: " $query
run "clickhouse benchmark <<< '$query' -c 8" &
echo "Running queries"

pid=`pidof qemu-system-x86_64`
sec=$(( (RANDOM % 5) + 25 ))
ms=$(( RANDOM % 1000 ))

echo "Will kill VM in $sec.$ms sec"

sleep $sec.$ms
kill -9 $pid

echo "Restarting"

sleep 5s

./startup.exp > qemu.log 2>&1 &
wait_vm_for_start

run "rm -r *data/system"
run "clickhouse server --config-file=/etc/clickhouse-server/config.xml > clickhouse-server.log 2>&1" &
wait_clickhouse_for_start

pid=`pidof qemu-system-x86_64`
result=`run "grep $TABLE_NAME clickhouse-server.log | grep 'Caught exception while loading metadata'"`
if [[ -n $result ]]; then
    echo "FAIL. Can't attach table:"
    echo $result
    kill -9 $pid
    exit 1
fi

result=`run "grep $TABLE_NAME clickhouse-server.log | grep 'Considering to remove broken part'"`
if [[ -n $result ]]; then
    echo "FAIL. Have broken parts:"
    echo $result
    kill -9 $pid
    exit 1
fi

kill -9 $pid
echo OK