mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Add retries into 01675_distributed_bytes_to_delay_insert (and tune delays)
This commit is contained in:
Azat Khuzhin
parent
f1187aeb69
commit
416bacc341
@ -1,3 +1,2 @@
|
||||
max_delay_to_insert will throw
|
||||
max_delay_to_insert will succeed
|
||||
flushed
|
||||
|
||||
@ -6,81 +6,123 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
max_delay_to_insert=5
|
||||
|
||||
${CLICKHOUSE_CLIENT} -nq "
|
||||
drop table if exists dist_01675;
|
||||
drop table if exists data_01675;
|
||||
"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -nq "
|
||||
create table data_01675 (key Int) engine=Null();
|
||||
create table dist_01675 (key Int) engine=Distributed(test_shard_localhost, currentDatabase(), data_01675) settings bytes_to_delay_insert=1, max_delay_to_insert=$max_delay_to_insert;
|
||||
system stop distributed sends dist_01675;
|
||||
"
|
||||
function drop_tables()
|
||||
{
|
||||
${CLICKHOUSE_CLIENT} -nq "
|
||||
drop table if exists dist_01675;
|
||||
drop table if exists data_01675;
|
||||
"
|
||||
}
|
||||
|
||||
#
|
||||
# Case 1: max_delay_to_insert will throw.
|
||||
#
|
||||
echo "max_delay_to_insert will throw"
|
||||
function test_max_delay_to_insert_will_throw()
|
||||
{
|
||||
echo "max_delay_to_insert will throw"
|
||||
|
||||
start_seconds=$SECONDS
|
||||
${CLICKHOUSE_CLIENT} --testmode -nq "
|
||||
-- first batch is always OK, since there is no pending bytes yet
|
||||
insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0;
|
||||
-- second will fail, because of bytes_to_delay_insert=1 and max_delay_to_insert=5,
|
||||
-- while distributed sends is stopped.
|
||||
--
|
||||
-- (previous block definitelly takes more, since it has header)
|
||||
insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0; -- { serverError 574 }
|
||||
system flush distributed dist_01675;
|
||||
"
|
||||
end_seconds=$SECONDS
|
||||
local max_delay_to_insert=2
|
||||
${CLICKHOUSE_CLIENT} -nq "
|
||||
create table data_01675 (key Int) engine=Null();
|
||||
create table dist_01675 (key Int) engine=Distributed(test_shard_localhost, currentDatabase(), data_01675) settings bytes_to_delay_insert=1, max_delay_to_insert=$max_delay_to_insert;
|
||||
system stop distributed sends dist_01675;
|
||||
"
|
||||
|
||||
if (( (end_seconds-start_seconds)<(max_delay_to_insert-1) )); then
|
||||
local start_seconds=$SECONDS
|
||||
${CLICKHOUSE_CLIENT} --testmode -nq "
|
||||
-- first batch is always OK, since there is no pending bytes yet
|
||||
insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0;
|
||||
-- second will fail, because of bytes_to_delay_insert=1 and max_delay_to_insert>0,
|
||||
-- while distributed sends is stopped.
|
||||
--
|
||||
-- (previous block definitelly takes more, since it has header)
|
||||
insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0; -- { serverError 574 }
|
||||
system flush distributed dist_01675;
|
||||
"
|
||||
local end_seconds=$SECONDS
|
||||
|
||||
if (( (end_seconds-start_seconds)<(max_delay_to_insert-1) )); then
|
||||
echo "max_delay_to_insert was not satisfied ($end_seconds-$start_seconds)"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
#
|
||||
# Case 2: max_delay_to_insert will finally finished.
|
||||
#
|
||||
echo "max_delay_to_insert will succeed"
|
||||
|
||||
max_delay_to_insert=10
|
||||
${CLICKHOUSE_CLIENT} -nq "
|
||||
drop table dist_01675;
|
||||
create table dist_01675 (key Int) engine=Distributed(test_shard_localhost, currentDatabase(), data_01675) settings bytes_to_delay_insert=1, max_delay_to_insert=$max_delay_to_insert;
|
||||
system stop distributed sends dist_01675;
|
||||
"
|
||||
|
||||
flush_delay=4
|
||||
function flush_distributed_worker()
|
||||
function test_max_delay_to_insert_will_succeed_once()
|
||||
{
|
||||
sleep $flush_delay
|
||||
local flush_delay=$1 && shift
|
||||
local max_delay_to_insert=$1 && shift
|
||||
|
||||
drop_tables
|
||||
|
||||
${CLICKHOUSE_CLIENT} -nq "
|
||||
create table data_01675 (key Int) engine=Null();
|
||||
create table dist_01675 (key Int) engine=Distributed(test_shard_localhost, currentDatabase(), data_01675) settings bytes_to_delay_insert=1, max_delay_to_insert=$max_delay_to_insert;
|
||||
system stop distributed sends dist_01675;
|
||||
"
|
||||
|
||||
function flush_distributed_worker()
|
||||
{
|
||||
sleep "$flush_delay"
|
||||
${CLICKHOUSE_CLIENT} -q "system flush distributed dist_01675"
|
||||
echo flushed
|
||||
}
|
||||
flush_distributed_worker &
|
||||
|
||||
local start_seconds=$SECONDS
|
||||
${CLICKHOUSE_CLIENT} --testmode -nq "
|
||||
-- first batch is always OK, since there is no pending bytes yet
|
||||
insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0;
|
||||
-- second will succeed, due to SYSTEM FLUSH DISTRIBUTED in background.
|
||||
insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0;
|
||||
"
|
||||
local end_seconds=$SECONDS
|
||||
wait
|
||||
|
||||
echo $((end_seconds-start_seconds))
|
||||
}
|
||||
flush_distributed_worker &
|
||||
function test_max_delay_to_insert_will_succeed()
|
||||
{
|
||||
echo "max_delay_to_insert will succeed"
|
||||
|
||||
start_seconds=$SECONDS
|
||||
${CLICKHOUSE_CLIENT} --testmode -nq "
|
||||
-- first batch is always OK, since there is no pending bytes yet
|
||||
insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0;
|
||||
-- second will succcedd, due to SYSTEM FLUSH DISTRIBUTED in background.
|
||||
insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0;
|
||||
"
|
||||
end_seconds=$SECONDS
|
||||
wait
|
||||
local max_delay_to_insert=4
|
||||
local flush_delay=2
|
||||
local diff
|
||||
local retries=20 i=0
|
||||
|
||||
if (( (end_seconds-start_seconds)<(flush_delay-1) )); then
|
||||
echo "max_delay_to_insert was not wait flush_delay ($end_seconds-$start_seconds)"
|
||||
fi
|
||||
if (( (end_seconds-start_seconds)>=(max_delay_to_insert-1) )); then
|
||||
echo "max_delay_to_insert was overcommited ($end_seconds-$start_seconds)"
|
||||
fi
|
||||
while (( (i++) < retries )); do
|
||||
diff=$(test_max_delay_to_insert_will_succeed_once $flush_delay $max_delay_to_insert)
|
||||
|
||||
if (( diff<(flush_delay-1) )); then
|
||||
echo "max_delay_to_insert was not wait flush_delay ($diff)"
|
||||
break
|
||||
fi
|
||||
|
||||
${CLICKHOUSE_CLIENT} -nq "
|
||||
drop table dist_01675;
|
||||
drop table data_01675;
|
||||
"
|
||||
# retry the test until the diff will be satisfied
|
||||
# (since we cannot assume that there will be no other lags)
|
||||
if (( diff>=(max_delay_to_insert-1) )); then
|
||||
continue
|
||||
fi
|
||||
|
||||
return
|
||||
done
|
||||
|
||||
echo "max_delay_to_insert was overcommited ($diff)"
|
||||
}
|
||||
|
||||
function run_test()
|
||||
{
|
||||
local test_case=$1 && shift
|
||||
|
||||
drop_tables
|
||||
$test_case
|
||||
}
|
||||
|
||||
function main()
|
||||
{
|
||||
run_test test_max_delay_to_insert_will_throw
|
||||
run_test test_max_delay_to_insert_will_succeed
|
||||
|
||||
drop_tables
|
||||
}
|
||||
main "$@"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user