ClickHouse/tests/queries/0_stateless/01154_move_partition_long.sh

132 lines
4.5 KiB
Bash
Raw Normal View History

2021-05-10 18:03:37 +00:00
#!/usr/bin/env bash
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
declare -A engines
engines[0]="MergeTree"
2021-06-01 16:26:54 +00:00
engines[1]="ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/{shard}/src', '{replica}_' || toString(randConstant()))"
engines[2]="ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/{shard}/src_' || toString(randConstant()), '{replica}')"
2021-05-10 18:03:37 +00:00
for ((i=0; i<16; i++)) do
$CLICKHOUSE_CLIENT -q "CREATE TABLE dst_$i (p UInt64, k UInt64, v UInt64)
ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/dst', '$i')
2021-06-21 17:29:32 +00:00
PARTITION BY p % 10 ORDER BY k" 2>&1| grep -Pv "Retrying createReplica|created by another server at the same moment, will retry|is already started to be removing" 2>&1 &
2021-05-10 18:03:37 +00:00
engine=${engines[$((i % ${#engines[@]}))]}
$CLICKHOUSE_CLIENT -q "CREATE TABLE src_$i (p UInt64, k UInt64, v UInt64) ENGINE=$engine
2021-06-21 17:29:32 +00:00
PARTITION BY p % 10 ORDER BY k" 2>&1| grep -Pv "Retrying createReplica|created by another server at the same moment, will retry|is already started to be removing" 2>&1 &
2021-05-10 18:03:37 +00:00
done
wait
2021-05-11 12:31:13 +00:00
#function create_drop_thread()
#{
# while true; do
# REPLICA=$(($RANDOM % 16))
# $CLICKHOUSE_CLIENT -q "DROP TABLE src_$REPLICA;"
# arr=("$@")
# engine=${arr[$RANDOM % ${#arr[@]}]}
# $CLICKHOUSE_CLIENT -q "CREATE TABLE src_$REPLICA (p UInt64, k UInt64, v UInt64) ENGINE=$engine PARTITION BY p % 10 ORDER BY k"
# sleep 0.$RANDOM;
# done
#}
2021-05-10 18:03:37 +00:00
function insert_thread()
{
while true; do
REPLICA=$(($RANDOM % 16))
LIMIT=$(($RANDOM % 100))
$CLICKHOUSE_CLIENT -q "INSERT INTO $1_$REPLICA SELECT * FROM generateRandom('p UInt64, k UInt64, v UInt64') LIMIT $LIMIT" 2>/dev/null
done
}
function move_partition_src_dst_thread()
{
while true; do
FROM_REPLICA=$(($RANDOM % 16))
TO_REPLICA=$(($RANDOM % 16))
PARTITION=$(($RANDOM % 10))
$CLICKHOUSE_CLIENT -q "ALTER TABLE src_$FROM_REPLICA MOVE PARTITION $PARTITION TO TABLE dst_$TO_REPLICA" 2>/dev/null
sleep 0.$RANDOM;
done
}
function replace_partition_src_src_thread()
{
while true; do
FROM_REPLICA=$(($RANDOM % 16))
TO_REPLICA=$(($RANDOM % 16))
PARTITION=$(($RANDOM % 10))
$CLICKHOUSE_CLIENT -q "ALTER TABLE src_$TO_REPLICA REPLACE PARTITION $PARTITION FROM src_$FROM_REPLICA" 2>/dev/null
sleep 0.$RANDOM;
done
}
function drop_partition_thread()
{
while true; do
REPLICA=$(($RANDOM % 16))
PARTITION=$(($RANDOM % 10))
2021-05-13 11:29:59 +00:00
$CLICKHOUSE_CLIENT -q "ALTER TABLE dst_$REPLICA DROP PARTITION $PARTITION" 2>/dev/null
2021-05-10 18:03:37 +00:00
sleep 0.$RANDOM;
done
}
function optimize_thread()
{
while true; do
REPLICA=$(($RANDOM % 16))
2021-05-14 12:55:30 +00:00
TABLE="src"
2021-05-10 18:03:37 +00:00
if (( RANDOM % 2 )); then
2021-05-14 12:55:30 +00:00
TABLE="dst"
2021-05-10 18:03:37 +00:00
fi
$CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE ${TABLE}_$REPLICA" 2>/dev/null
sleep 0.$RANDOM;
done
}
2021-07-06 16:51:23 +00:00
function drop_part_thread()
{
while true; do
REPLICA=$(($RANDOM % 16))
part=$($CLICKHOUSE_CLIENT -q "SELECT name FROM system.parts WHERE active AND database='$CLICKHOUSE_DATABASE' and table='dst_$REPLICA' ORDER BY rand() LIMIT 1")
$CLICKHOUSE_CLIENT -q "ALTER TABLE dst_$REPLICA DROP PART '$part'" 2>/dev/null
sleep 0.$RANDOM;
done
}
2021-05-11 12:31:13 +00:00
#export -f create_drop_thread;
2021-05-10 18:03:37 +00:00
export -f insert_thread;
export -f move_partition_src_dst_thread;
export -f replace_partition_src_src_thread;
export -f drop_partition_thread;
export -f optimize_thread;
2021-07-06 16:51:23 +00:00
export -f drop_part_thread;
2021-05-10 18:03:37 +00:00
2021-05-13 14:23:00 +00:00
TIMEOUT=60
2021-05-10 18:03:37 +00:00
2021-05-11 12:31:13 +00:00
#timeout $TIMEOUT bash -c "create_drop_thread ${engines[@]}" &
2021-05-10 18:03:37 +00:00
timeout $TIMEOUT bash -c 'insert_thread src' &
timeout $TIMEOUT bash -c 'insert_thread src' &
timeout $TIMEOUT bash -c 'insert_thread dst' &
timeout $TIMEOUT bash -c move_partition_src_dst_thread &
timeout $TIMEOUT bash -c replace_partition_src_src_thread &
timeout $TIMEOUT bash -c drop_partition_thread &
timeout $TIMEOUT bash -c optimize_thread &
2021-07-06 16:51:23 +00:00
timeout $TIMEOUT bash -c drop_part_thread &
2021-05-10 18:03:37 +00:00
wait
for ((i=0; i<16; i++)) do
2021-07-21 01:05:56 +00:00
# The size of log is big, so increase timeout.
$CLICKHOUSE_CLIENT --receive_timeout 600 -q "SYSTEM SYNC REPLICA dst_$i" &
$CLICKHOUSE_CLIENT --receive_timeout 600 -q "SYSTEM SYNC REPLICA src_$i" 2>/dev/null &
2021-05-10 18:03:37 +00:00
done
2021-05-14 12:55:30 +00:00
wait
2021-05-10 18:03:37 +00:00
echo "Replication did not hang"
for ((i=0; i<16; i++)) do
2021-07-29 13:40:33 +00:00
$CLICKHOUSE_CLIENT -q "DROP TABLE dst_$i" 2>&1| grep -Fv "is already started to be removing" &
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS src_$i" 2>&1| grep -Fv "is already started to be removing" &
2021-05-10 18:03:37 +00:00
done
wait