ClickHouse/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh

135 lines
4.8 KiB
Bash
Raw Normal View History

2020-02-05 16:30:02 +00:00
#!/usr/bin/env bash
2021-09-12 12:35:27 +00:00
# Tags: replica, no-parallel, no-fasttest
2020-02-05 16:30:02 +00:00
# This test checks mutations concurrent execution with concurrent inserts.
# There was a bug in mutations finalization, when mutation finishes not after all
# MUTATE_PART tasks execution, but after GET of already mutated part from other replica.
2020-02-20 08:50:00 +00:00
# To test it we stop some replicas to delay fetch of required parts for mutation.
2020-02-05 16:30:02 +00:00
# Since our replication queue executing tasks concurrently it may happen, that we dowload already mutated
# part before source part.
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
2020-12-28 11:46:53 +00:00
# shellcheck source=../shell_config.sh
2020-08-01 00:51:12 +00:00
. "$CURDIR"/../shell_config.sh
2020-02-05 16:30:02 +00:00
REPLICAS=5
for i in $(seq $REPLICAS); do
2020-02-19 20:44:36 +00:00
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_mutate_mt_$i"
2020-02-05 16:30:02 +00:00
done
for i in $(seq $REPLICAS); do
Do not overlap zookeeper path for ReplicatedMergeTree in stateless *.sh tests Found with stress tests for 00626_replace_partition_from_table_zookeeper [1]: 2021.03.15 00:59:48.200106 [ 27417 ] {0f47dbeb-938a-4560-8408-a7cc7b3bafb8} <Trace> ContextAccess (default): Access granted: CREATE TABLE ON test_31.dst_r1 ... 2021.03.15 00:59:48.403227 [ 27417 ] {0f47dbeb-938a-4560-8408-a7cc7b3bafb8} <Debug> test_31.dst_r1: This table /clickhouse/test_00626/dst_1 is already created, will add new replica 2021.03.15 00:59:48.736450 [ 83006 ] {b2db1355-3ec3-4e3a-9c79-f93f27c6e658} <Trace> ContextAccess (default): Access granted: CREATE TABLE ON test_31.dst_r2 ... 2021.03.15 00:59:48.851768 [ 83006 ] {b2db1355-3ec3-4e3a-9c79-f93f27c6e658} <Debug> test_31.dst_r2: This table /clickhouse/test_00626/dst_1 is already created, will add new replica ... 2021.03.15 00:59:48.919059 [ 366 ] {} <Debug> test_31.dst_r2 (ReplicatedMergeTreeQueue): Loading queue from /clickhouse/test_00626/dst_1/replicas/2/queue 2021.03.15 00:59:48.919948 [ 366 ] {} <Debug> test_31.dst_r2 (ReplicatedMergeTreeQueue): Having 3 queue entries to load, 0 entries already loaded. 2021.03.15 00:59:48.921833 [ 366 ] {} <Trace> test_31.dst_r2 (ReplicatedMergeTreeQueue): Loaded queue ... 2021.03.15 00:59:51.904230 [ 246952 ] {59753eea-3896-45ca-8625-fdaa094ee9ef} <Trace> ContextAccess (default): Access granted: SYSTEM SYNC REPLICA ON test_31.dst_r2 ... 2021.03.15 01:04:51.913683 [ 246952 ] {59753eea-3896-45ca-8625-fdaa094ee9ef} <Error> InterpreterSystemQuery: SYNC REPLICA test_31.dst_r2: Timed out! [1]: https://clickhouse-test-reports.s3.yandex.net/21716/402bf77783cbda48a9ee1b748bfce3c52ef8fe11/stress_test_(memory)/test_run.txt.out.log But the problem is more generic, so fix all tests.
2021-03-15 04:51:28 +00:00
$CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_mutate_mt_$i (key UInt64, value1 UInt64, value2 String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_mutate_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0"
2020-02-05 16:30:02 +00:00
done
2020-02-19 20:44:36 +00:00
$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_mt_1 SELECT number, number + 10, toString(number) from numbers(10)"
$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_mt_1 SELECT number, number + 10, toString(number) from numbers(10, 40)"
2020-02-05 16:30:02 +00:00
for i in $(seq $REPLICAS); do
2020-02-19 20:44:36 +00:00
$CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_mutate_mt_$i"
2020-02-05 16:30:02 +00:00
done
for i in $(seq $REPLICAS); do
2020-02-19 20:44:36 +00:00
$CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_mutate_mt_$i"
2020-02-05 16:30:02 +00:00
done
INITIAL_SUM=$($CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_mutate_mt_1")
2020-02-05 16:30:02 +00:00
# Run mutation on random replica
function correct_alter_thread()
{
while true; do
REPLICA=$(($RANDOM % 5 + 1))
2020-02-19 20:44:36 +00:00
$CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_mt_$REPLICA UPDATE value1 = value1 + 1 WHERE 1";
2020-03-02 10:19:16 +00:00
sleep 1
2020-02-05 16:30:02 +00:00
done
}
# This thread add some data to table.
function insert_thread()
{
VALUES=(7 8 9)
while true; do
REPLICA=$(($RANDOM % 5 + 1))
VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]}
2020-02-19 20:44:36 +00:00
$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_mt_$REPLICA VALUES($RANDOM, $VALUE, toString($VALUE))"
2020-02-05 16:30:02 +00:00
sleep 0.$RANDOM
done
}
2020-02-19 20:44:36 +00:00
function detach_attach_thread()
2020-02-05 16:30:02 +00:00
{
while true; do
REPLICA=$(($RANDOM % 5 + 1))
2020-02-19 20:44:36 +00:00
$CLICKHOUSE_CLIENT --query "DETACH TABLE concurrent_mutate_mt_$REPLICA"
2020-02-05 16:30:02 +00:00
sleep 0.$RANDOM
2020-02-20 08:50:00 +00:00
sleep 0.$RANDOM
sleep 0.$RANDOM
2020-02-19 20:44:36 +00:00
$CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_mutate_mt_$REPLICA"
2020-02-05 16:30:02 +00:00
done
}
echo "Starting alters"
export -f correct_alter_thread;
export -f insert_thread;
2020-02-19 20:44:36 +00:00
export -f detach_attach_thread;
2020-02-05 16:30:02 +00:00
2020-02-20 08:50:00 +00:00
# We assign a lot of mutations so timeout shouldn't be too big
TIMEOUT=15
2020-02-05 16:30:02 +00:00
2020-02-19 20:44:36 +00:00
timeout $TIMEOUT bash -c detach_attach_thread 2> /dev/null &
2020-02-05 16:30:02 +00:00
timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null &
timeout $TIMEOUT bash -c insert_thread 2> /dev/null &
timeout $TIMEOUT bash -c insert_thread 2> /dev/null &
timeout $TIMEOUT bash -c insert_thread 2> /dev/null &
timeout $TIMEOUT bash -c insert_thread 2> /dev/null &
timeout $TIMEOUT bash -c insert_thread 2> /dev/null &
timeout $TIMEOUT bash -c insert_thread 2> /dev/null &
timeout $TIMEOUT bash -c insert_thread 2> /dev/null &
wait
echo "Finishing alters"
for i in $(seq $REPLICAS); do
2020-02-19 20:44:36 +00:00
$CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_mutate_mt_$i" 2> /dev/null
2020-02-05 16:30:02 +00:00
done
2020-02-19 20:44:36 +00:00
sleep 1
2020-02-05 16:30:02 +00:00
2020-02-20 08:50:00 +00:00
counter=0
2020-12-21 08:48:40 +00:00
have_undone_mutations_query="select * from system.mutations where table like 'concurrent_mutate_mt_%' and is_done=0 and database='${CLICKHOUSE_DATABASE}'"
have_all_tables_query="select count() FROM system.tables WHERE name LIKE 'concurrent_mutate_mt_%' and database='${CLICKHOUSE_DATABASE}'"
2020-02-20 08:50:00 +00:00
2020-12-21 08:48:40 +00:00
while true ; do
2020-06-15 19:11:17 +00:00
if [ "$counter" -gt 120 ]
2020-02-20 08:50:00 +00:00
then
break
fi
2020-02-19 20:44:36 +00:00
sleep 1
for i in $(seq $REPLICAS); do
2020-06-16 10:34:59 +00:00
$CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_mutate_mt_$i" 2> /dev/null
done
2020-12-21 08:48:40 +00:00
2020-02-20 08:50:00 +00:00
counter=$(($counter + 1))
2020-12-21 08:48:40 +00:00
# no active mutations and all tables attached
if [[ -z $($CLICKHOUSE_CLIENT --query "$have_undone_mutations_query" 2>&1) && $($CLICKHOUSE_CLIENT --query "$have_all_tables_query" 2>&1) == "$REPLICAS" ]]; then
break
fi
2020-02-05 16:30:02 +00:00
done
for i in $(seq $REPLICAS); do
2020-02-19 20:44:36 +00:00
$CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) > $INITIAL_SUM FROM concurrent_mutate_mt_$i"
$CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE table='concurrent_mutate_mt_$i' and is_done=0" # all mutations have to be done
$CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE table='concurrent_mutate_mt_$i' and is_done=0" # for verbose output
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_mutate_mt_$i"
2020-02-05 16:30:02 +00:00
done