add test for durability (draft)

utils/durability-test/create.sql

@@ -0,0 +1 @@
+CREATE TABLE test (a Int, s String) ENGINE = MergeTree ORDER BY a;

utils/durability-test/durability-test.sh

@@ -0,0 +1,154 @@
+#!/bin/bash
+
+URL=http://cloud-images.ubuntu.com/bionic/current
+IMAGE=bionic-server-cloudimg-amd64.img
+SSH_PORT=11022
+CLICKHOUSE_PORT=9090
+PASSWORD=root
+
+TABLE_NAME=$1
+CREATE_QUERY=$2
+INSERT_QUERY=$3
+
+if [[ -z $TABLE_NAME || -z $CREATE_QUERY || -z $INSERT_QUERY ]]; then
+    echo "Required 3 arguments: table name, file with create query, file with insert query"
+    exit 1
+fi
+
+function run()
+{
+    sshpass -p $PASSWORD ssh -p $SSH_PORT root@localhost "$1"
+}
+
+function copy()
+{
+    sshpass -p $PASSWORD scp -r -P $SSH_PORT $1 root@localhost:$2
+}
+
+function wait_vm_for_start()
+{
+    echo "Waiting until VM started..."
+    started=0
+    for i in {0..100}; do
+        run "exit"
+        if [ $? -eq 0 ]; then
+            started=1
+            break
+        fi 
+        sleep 1s
+    done
+
+    if ((started == 0)); then
+        echo "Can't start or connect to VM."
+        exit 1
+    fi
+
+    echo "Started VM"
+}
+
+function wait_clickhouse_for_start()
+{
+    echo "Waiting until ClickHouse started..."
+    started=0
+    for i in {0..15}; do
+        run "clickhouse client --query 'select 1'"
+        if [ $? -eq 0 ]; then
+            started=1
+            break
+        fi
+        sleep 1s
+    done
+
+    if ((started == 0)); then
+        echo "Can't start ClickHouse."
+    fi
+
+    echo "Started ClickHouse"
+}
+
+echo "Downloading image"
+curl -O $URL/$IMAGE
+
+qemu-img resize $IMAGE +10G
+virt-customize -a $IMAGE --root-password password:$PASSWORD
+virt-copy-in -a $IMAGE sshd_config /etc/ssh
+
+echo "Starting VM"
+
+chmod +x ./startup.exp
+./startup.exp > qemu.log 2>&1 &
+
+wait_vm_for_start
+
+echo "Preparing VM"
+
+# Resize partition
+run "growpart /dev/sda 1 && resize2fs /dev/sda1"
+
+if [[ -z $CLICKHOUSE_BINARY ]]; then
+    CLICKHOUSE_BINARY=/usr/bin/clickhouse
+fi
+
+if [[ -z $CLICKHOUSE_CONFIG_DIR ]]; then
+    CLICKHOUSE_CONFIG_DIR=/etc/clickhouse-server
+fi
+
+echo "Using ClickHouse binary: " $CLICKHOUSE_BINARY
+echo "Using ClickHouse config from: " $CLICKHOUSE_CONFIG_DIR
+
+copy $CLICKHOUSE_BINARY /usr/bin
+copy $CLICKHOUSE_CONFIG_DIR /etc
+run "mv /etc/$CLICKHOUSE_CONFIG_DIR /etc/clickhouse-server"
+
+echo "Prepared VM"
+echo "Starting ClickHouse"
+
+run "clickhouse server --config-file=/etc/clickhouse-server/config.xml > clickhouse-server.log 2>&1" &
+
+wait_clickhouse_for_start
+
+echo "Started ClickHouse"
+
+query=`cat $CREATE_QUERY`
+echo "Executing query:" $query
+run "clickhouse client --query '$query'"
+
+query=`cat $INSERT_QUERY`
+echo "Will run in a loop query: " $query
+run "clickhouse benchmark <<< '$query'" &
+echo "Running queries"
+
+pid=`pidof qemu-system-x86_64`
+sec=$(( (RANDOM % 3) + 25 ))
+
+ms=$(( RANDOM % 1000 ))
+
+echo "Will kill VM in $sec.$ms sec"
+
+sleep $sec.$ms
+kill -9 $pid
+
+echo "Restarting"
+
+./startup.exp > qemu.log 2>&1 &
+wait_vm_for_start
+
+run "rm -r *data/system"
+run "clickhouse server --config-file=/etc/clickhouse-server/config.xml > clickhouse-server.log 2>&1" &
+wait_clickhouse_for_start
+
+result=`run "grep $TABLE_NAME clickhouse-server.log | grep 'Caught exception while loading metadata'"`
+if [[ -n $result ]]; then
+    echo "FAIL. Can't attach table:"
+    echo $result
+    exit 1
+fi
+
+result=`run "grep $TABLE_NAME clickhouse-server.log | grep 'Considering to remove broken part'"`
+if [[ -n $result ]]; then
+    echo "FAIL. Have broken parts:"
+    echo $result
+    exit 1
+fi
+
+echo OK

utils/durability-test/insert.sql

@@ -0,0 +1 @@
+INSERT INTO test SELECT number, toString(number) FROM numbers(10)

utils/durability-test/install.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+apt update && apt install qemu-kvm qemu virt-manager virt-viewer libguestfs-tools sshpass expect

utils/durability-test/sshd_config

@@ -0,0 +1,8 @@
+PermitRootLogin yes
+PasswordAuthentication yes
+ChallengeResponseAuthentication no
+UsePAM yes
+X11Forwarding yes
+PrintMotd no
+AcceptEnv LANG LC_*
+Subsystem	sftp	/usr/lib/openssh/sftp-server

utils/durability-test/startup.exp

@@ -0,0 +1,23 @@
+#!/usr/bin/expect -f
+
+# Wait enough (forever) until a long-time boot
+set timeout -1
+
+spawn qemu-system-x86_64 \
+    -hda bionic-server-cloudimg-amd64.img \
+    -cpu qemu64,+ssse3,+sse4.1,+sse4.2,+popcnt -smp 8 \
+    -net nic -net user,hostfwd=tcp::11022-:22 \
+    -m 4096 -nographic
+
+expect "login: "
+send "root\n"
+
+expect "Password: "
+send "root\n"
+
+# Without it ssh is not working on guest machine for some reason
+expect "# "
+send "dhclient && ssh-keygen -A && systemctl restart sshd.service\n"
+
+# Wait forever
+expect "########"