mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 00:30:49 +00:00
Merge pull request #32492 from azat/fix-01675_distributed_bytes_to_delay_insert
Make 01675_distributed_bytes_to_delay_insert less flaky (use http over client)
This commit is contained in:
commit
ccd4d2fe3a
@ -1,2 +1,3 @@
|
||||
max_delay_to_insert will throw
|
||||
Too many bytes pending for async INSERT
|
||||
max_delay_to_insert will succeed
|
125
tests/queries/0_stateless/01675_distributed_bytes_to_delay_insert.sh
Executable file
125
tests/queries/0_stateless/01675_distributed_bytes_to_delay_insert.sh
Executable file
@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long, distributed
|
||||
|
||||
# NOTE: $SECONDS accuracy is second, so we need some delta, hence -1 in time conditions.
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
function drop_tables()
|
||||
{
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d @- <<<"drop table if exists dist_01675"
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d @- <<<"drop table if exists data_01675"
|
||||
}
|
||||
|
||||
#
|
||||
# Case 1: max_delay_to_insert will throw.
|
||||
#
|
||||
function test_max_delay_to_insert_will_throw()
|
||||
{
|
||||
echo "max_delay_to_insert will throw"
|
||||
|
||||
local max_delay_to_insert=2
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d @- <<<"create table data_01675 (key Int) engine=Null()"
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d @- <<<"create table dist_01675 (key Int) engine=Distributed(test_shard_localhost, currentDatabase(), data_01675) settings bytes_to_delay_insert=1, max_delay_to_insert=$max_delay_to_insert"
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d @- <<<"system stop distributed sends dist_01675"
|
||||
|
||||
local start_seconds=$SECONDS
|
||||
# first batch is always OK, since there is no pending bytes yet
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d @- <<<"insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0"
|
||||
# second will fail, because of bytes_to_delay_insert=1 and max_delay_to_insert>0,
|
||||
# while distributed sends is stopped.
|
||||
#
|
||||
# (previous block definitelly takes more, since it has header)
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d @- <<<"insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0" |& grep -o 'Too many bytes pending for async INSERT'
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d @- <<<"system flush distributed dist_01675"
|
||||
local end_seconds=$SECONDS
|
||||
|
||||
if (( (end_seconds-start_seconds)<(max_delay_to_insert-1) )); then
|
||||
echo "max_delay_to_insert was not satisfied ($end_seconds-$start_seconds)"
|
||||
fi
|
||||
}
|
||||
|
||||
#
|
||||
# Case 2: max_delay_to_insert will finally finished.
|
||||
#
|
||||
function test_max_delay_to_insert_will_succeed_once()
|
||||
{
|
||||
local max_delay_to_insert=4
|
||||
local flush_delay=2
|
||||
|
||||
drop_tables
|
||||
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d @- <<<"create table data_01675 (key Int) engine=Null()"
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d @- <<<"create table dist_01675 (key Int) engine=Distributed(test_shard_localhost, currentDatabase(), data_01675) settings bytes_to_delay_insert=1, max_delay_to_insert=$max_delay_to_insert"
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d @- <<<"system stop distributed sends dist_01675"
|
||||
|
||||
function flush_distributed_worker()
|
||||
{
|
||||
sleep $flush_delay
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d @- <<<"system flush distributed dist_01675"
|
||||
}
|
||||
flush_distributed_worker &
|
||||
|
||||
local start_seconds=$SECONDS
|
||||
{
|
||||
# NOTE:
|
||||
# ignore stderr, since it may produce exception if flushing thread will be too slow
|
||||
# (this is possible on CI)
|
||||
|
||||
# first batch is always OK, since there is no pending bytes yet
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1" -d @- <<<"insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0" >& /dev/null
|
||||
# second will succeed, due to SYSTEM FLUSH DISTRIBUTED in background.
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1" -d @- <<<"insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0" >& /dev/null
|
||||
}
|
||||
local end_seconds=$SECONDS
|
||||
|
||||
wait
|
||||
|
||||
local diff=$(( end_seconds-start_seconds ))
|
||||
|
||||
if (( diff<(flush_delay-1) )); then
|
||||
# this is fatal error, that should not be retriable
|
||||
echo "max_delay_to_insert was not wait flush_delay ($diff)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# retry the test until the diff will be satisfied
|
||||
# (since we should not assume that there will be no other lags)
|
||||
if (( diff>=(max_delay_to_insert-1) )); then
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
function test_max_delay_to_insert_will_succeed()
|
||||
{
|
||||
echo "max_delay_to_insert will succeed"
|
||||
|
||||
local retries=20 i=0
|
||||
while (( (i++) < retries )); do
|
||||
if test_max_delay_to_insert_will_succeed_once; then
|
||||
return
|
||||
fi
|
||||
done
|
||||
|
||||
echo failed
|
||||
}
|
||||
|
||||
function run_test()
|
||||
{
|
||||
local test_case=$1 && shift
|
||||
|
||||
drop_tables
|
||||
$test_case
|
||||
}
|
||||
|
||||
function main()
|
||||
{
|
||||
run_test test_max_delay_to_insert_will_throw
|
||||
run_test test_max_delay_to_insert_will_succeed
|
||||
|
||||
drop_tables
|
||||
}
|
||||
main "$@"
|
@ -1,130 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long, distributed
|
||||
|
||||
# NOTE: $SECONDS accuracy is second, so we need some delta, hence -1 in time conditions.
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
function drop_tables()
|
||||
{
|
||||
${CLICKHOUSE_CLIENT} -nq "
|
||||
drop table if exists dist_01675;
|
||||
drop table if exists data_01675;
|
||||
"
|
||||
}
|
||||
|
||||
#
|
||||
# Case 1: max_delay_to_insert will throw.
|
||||
#
|
||||
function test_max_delay_to_insert_will_throw()
|
||||
{
|
||||
echo "max_delay_to_insert will throw"
|
||||
|
||||
local max_delay_to_insert=2
|
||||
${CLICKHOUSE_CLIENT} -nq "
|
||||
create table data_01675 (key Int) engine=Null();
|
||||
create table dist_01675 (key Int) engine=Distributed(test_shard_localhost, currentDatabase(), data_01675) settings bytes_to_delay_insert=1, max_delay_to_insert=$max_delay_to_insert;
|
||||
system stop distributed sends dist_01675;
|
||||
"
|
||||
|
||||
local start_seconds=$SECONDS
|
||||
${CLICKHOUSE_CLIENT} --testmode -nq "
|
||||
-- first batch is always OK, since there is no pending bytes yet
|
||||
insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0;
|
||||
-- second will fail, because of bytes_to_delay_insert=1 and max_delay_to_insert>0,
|
||||
-- while distributed sends is stopped.
|
||||
--
|
||||
-- (previous block definitelly takes more, since it has header)
|
||||
insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0; -- { serverError 574 }
|
||||
system flush distributed dist_01675;
|
||||
"
|
||||
local end_seconds=$SECONDS
|
||||
|
||||
if (( (end_seconds-start_seconds)<(max_delay_to_insert-1) )); then
|
||||
echo "max_delay_to_insert was not satisfied ($end_seconds-$start_seconds)"
|
||||
fi
|
||||
}
|
||||
|
||||
#
|
||||
# Case 2: max_delay_to_insert will finally finished.
|
||||
#
|
||||
function test_max_delay_to_insert_will_succeed_once()
|
||||
{
|
||||
local max_delay_to_insert=4
|
||||
local flush_delay=2
|
||||
|
||||
drop_tables
|
||||
|
||||
${CLICKHOUSE_CLIENT} -nq "
|
||||
create table data_01675 (key Int) engine=Null();
|
||||
create table dist_01675 (key Int) engine=Distributed(test_shard_localhost, currentDatabase(), data_01675) settings bytes_to_delay_insert=1, max_delay_to_insert=$max_delay_to_insert;
|
||||
system stop distributed sends dist_01675;
|
||||
"
|
||||
|
||||
function flush_distributed_worker()
|
||||
{
|
||||
sleep $flush_delay
|
||||
${CLICKHOUSE_CLIENT} -q "system flush distributed dist_01675"
|
||||
}
|
||||
flush_distributed_worker &
|
||||
|
||||
local start_seconds=$SECONDS
|
||||
# ignore stderr, since it may produce exception if flushing thread will be too slow
|
||||
# (this is possible on CI)
|
||||
${CLICKHOUSE_CLIENT} --testmode -nq "
|
||||
-- first batch is always OK, since there is no pending bytes yet
|
||||
insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0;
|
||||
-- second will succeed, due to SYSTEM FLUSH DISTRIBUTED in background.
|
||||
insert into dist_01675 select * from numbers(1) settings prefer_localhost_replica=0;
|
||||
" >& /dev/null
|
||||
local end_seconds=$SECONDS
|
||||
wait
|
||||
|
||||
local diff=$(( end_seconds-start_seconds ))
|
||||
|
||||
if (( diff<(flush_delay-1) )); then
|
||||
# this is fatal error, that should not be retriable
|
||||
echo "max_delay_to_insert was not wait flush_delay ($diff)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# retry the test until the diff will be satisfied
|
||||
# (since we cannot assume that there will be no other lags)
|
||||
if (( diff>=(max_delay_to_insert-1) )); then
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
function test_max_delay_to_insert_will_succeed()
|
||||
{
|
||||
echo "max_delay_to_insert will succeed"
|
||||
|
||||
local retries=20 i=0
|
||||
while (( (i++) < retries )); do
|
||||
if test_max_delay_to_insert_will_succeed_once; then
|
||||
return
|
||||
fi
|
||||
done
|
||||
|
||||
echo failed
|
||||
}
|
||||
|
||||
function run_test()
|
||||
{
|
||||
local test_case=$1 && shift
|
||||
|
||||
drop_tables
|
||||
$test_case
|
||||
}
|
||||
|
||||
function main()
|
||||
{
|
||||
run_test test_max_delay_to_insert_will_throw
|
||||
run_test test_max_delay_to_insert_will_succeed
|
||||
|
||||
drop_tables
|
||||
}
|
||||
main "$@"
|
Loading…
Reference in New Issue
Block a user