Review fixes

This commit is contained in:
alesapin 2021-10-11 16:40:12 +03:00
parent 59a78830f9
commit 3ae960e04b
11 changed files with 42 additions and 23 deletions

View File

@ -117,7 +117,7 @@ def prepare_for_hung_check(drop_databases):
# Wait for last queries to finish if any, not longer than 300 seconds
call("""clickhouse client -q "select sleepEachRow((
select maxOrDefault(300 - elapsed) + 1 from system.processes where query not like '%from system.processes%' and elapsed < 300
) / 300) from numbers(300) format Null" """, shell=True, stderr=STDOUT, timeout=30)
) / 300) from numbers(300) format Null" """, shell=True, stderr=STDOUT, timeout=330)
# Even if all clickhouse-test processes are finished, there are probably some sh scripts,
# which still run some new queries. Let's ignore them.

View File

@ -879,7 +879,7 @@ def run_tests_array(all_tests_with_params):
while True:
if is_concurrent:
case = queue.get(timeout=args.timeout)
case = queue.get(timeout=args.timeout * 1.1)
if not case:
break
else:
@ -1080,10 +1080,10 @@ def do_run_tests(jobs, test_suite: TestSuite, parallel):
pool.map_async(run_tests_array, parallel_tests_array)
for suit in test_suite.parallel_tests:
queue.put(suit, timeout=args.timeout)
queue.put(suit, timeout=args.timeout * 1.1)
for _ in range(jobs):
queue.put(None, timeout=args.timeout)
queue.put(None, timeout=args.timeout * 1.1)
queue.close()

View File

@ -7,9 +7,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
${CLICKHOUSE_CURL} --max-time 1 -sS "${CLICKHOUSE_URL}&query_id=cancel_http_readonly_queries_on_client_close&cancel_http_readonly_queries_on_client_close=1&query=SELECT+count()+FROM+system.numbers" 2>&1 | grep -cF 'curl: (28)'
while true
do
i=0 retries=300
while [[ $i -lt $retries ]]; do
${CLICKHOUSE_CURL} -sS --data "SELECT count() FROM system.processes WHERE query_id = 'cancel_http_readonly_queries_on_client_close'" "${CLICKHOUSE_URL}" | grep '0' && break
((++i))
sleep 0.2
done

View File

@ -5,10 +5,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
i=0 retries=300
# Sometimes five seconds are not enough due to system overload.
# But if it can run in less than five seconds at least sometimes - it is enough for the test.
while true
do
while [[ $i -lt $retries ]]; do
opts=(
--max_distributed_connections 20
--max_threads 1
@ -19,4 +19,5 @@ do
# "$@" left to pass manual options (like --experimental_use_processors 0) during manual testing
timeout 10s ${CLICKHOUSE_CLIENT} "${opts[@]}" "$@" && break
((++i))
done

View File

@ -8,9 +8,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# Sometimes 1.8 seconds are not enough due to system overload.
# But if it can run in less than five seconds at least sometimes - it is enough for the test.
while true
do
i=0 retries=100
while [[ $i -lt $retries ]]; do
query="SELECT sleepEachRow(1) FROM remote('127.{2,3}', system.one) FORMAT Null"
# 1.8 less then 2 seconds, but long enough to cover possible load peaks
timeout 1.8s ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_distributed_connections=2&max_threads=1" -d "$query" && break
((++i))
done

View File

@ -21,7 +21,9 @@ CREATE QUOTA quota_by_forwarded_ip_${CLICKHOUSE_DATABASE} KEYED BY forwarded_ip_
echo '--- Test with quota by immediate IP ---'
while true; do
i=0 retries=300
while [[ $i -lt $retries ]]; do
((++i))
${CLICKHOUSE_CURL} --fail -sS "${CLICKHOUSE_URL}&user=quoted_by_ip_${CLICKHOUSE_DATABASE}" -d "SELECT count() FROM numbers(10)" 2>/dev/null || break
done | uniq
@ -33,14 +35,18 @@ ${CLICKHOUSE_CURL} -H 'X-Forwarded-For: 1.2.3.4' -sS "${CLICKHOUSE_URL}&user=quo
echo '--- Test with quota by forwarded IP ---'
while true; do
i=0 retries=300
while [[ $i -lt $retries ]]; do
((++i))
${CLICKHOUSE_CURL} --fail -sS "${CLICKHOUSE_URL}&user=quoted_by_forwarded_ip_${CLICKHOUSE_DATABASE}" -d "SELECT count() FROM numbers(10)" 2>/dev/null || break
done | uniq
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=quoted_by_forwarded_ip_${CLICKHOUSE_DATABASE}" -d "SELECT count() FROM numbers(10)" | grep -oF 'exceeded'
i=0 retries=300
# X-Forwarded-For is respected for quota by forwarded IP address
while true; do
while [[ $i -lt $retries ]]; do
((++i))
${CLICKHOUSE_CURL} -H 'X-Forwarded-For: 1.2.3.4' -sS "${CLICKHOUSE_URL}&user=quoted_by_forwarded_ip_${CLICKHOUSE_DATABASE}" -d "SELECT count() FROM numbers(10)" | grep -oP '^10$' || break
done | uniq

View File

@ -13,14 +13,18 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# If concurrency is 10 (good), the query may take less than 10 second with non-zero probability
# and the following loops will finish with probability 1 assuming independent random variables.
while true; do
i=0 retries=30
while [[ $i -lt $retries ]]; do
timeout 10 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 10 --query "
SELECT sleep(1.5) FROM remote('127.{1..10}', system.one) FORMAT Null" --prefer_localhost_replica=0 && break
((++i))
done
while true; do
i=0 retries=30
while [[ $i -lt $retries ]]; do
timeout 10 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 10 --query "
SELECT sleep(1.5) FROM remote('127.{1..10}', system.one) FORMAT Null" --prefer_localhost_replica=1 && break
((++i))
done
# If max_distributed_connections is low and async_socket_for_remote is disabled,

View File

@ -4,12 +4,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
counter=0 retries=60
I=0
while true
do
while [[ $counter -lt $retries ]]; do
I=$((I + 1))
TYPE=$(perl -e "print 'Array(' x $I; print 'UInt8'; print ')' x $I")
${CLICKHOUSE_CLIENT} --max_parser_depth 1000000 --query "SELECT * FROM remote('127.0.0.{1,2}', generateRandom('x $TYPE', 1, 1, 1)) LIMIT 1 FORMAT Null" 2>&1 | grep -q -F 'Maximum parse depth' && break;
((++counter))
done
#echo "I = ${I}"

View File

@ -13,13 +13,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
M=1000000
while true
do
i=0 retries=300
while [[ $i -lt $retries ]]; do
$CLICKHOUSE_CLIENT --allow_hyperscan 1 --max_memory_usage $M --format Null --query "
SELECT [1, 2, 3, 11] = arraySort(multiMatchAllIndices('фабрикант', ['', 'рикан', 'а', 'f[a${RANDOM}e]b[ei]rl', 'ф[иа${RANDOM}эе]б[еэи][рпл]', 'афиукд', 'a[f${RANDOM}t],th', '^ф[аие${RANDOM}э]?б?[еэи]?$', 'бе${RANDOM}рлик', 'fa${RANDOM}b', 'фа[беьв]+е?[рл${RANDOM}ко]']))
" 2>&1 | grep -q 'Memory limit' || break;
M=$((M + 100000))
((++i))
done
echo 'Ok'

View File

@ -16,9 +16,10 @@ insert into aliases_lazyness(x) select * from numbers(40);
# The exact time is not guaranteed, so we check in a loop that at least once
# the query will process in less than one second, that proves that the behaviour is not like it was long time ago.
while true
do
i=0 retries=300
while [[ $i -lt $retries ]]; do
timeout 1 ${CLICKHOUSE_CLIENT} --query "SELECT x, y FROM aliases_lazyness WHERE x = 1 FORMAT Null" && break
((++i))
done
${CLICKHOUSE_CLIENT} --multiquery --query "

View File

@ -5,9 +5,11 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
i=0 retries=60
# Sometimes five seconds are not enough due to system overload.
# But if it can run in less than five seconds at least sometimes - it is enough for the test.
while true
do
while [[ $i -lt $retries ]]; do
timeout 5s ${CLICKHOUSE_CLIENT} --max_threads 10 --query "SELECT * FROM url('http://127.0.0.{1..10}:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+sleep(1)', TSV, 'x UInt8')" --format Null && break
((++i))
done