From abb747498d04fd6d6cac71d6097741b41bf572e3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 3 Aug 2024 16:54:44 +0200 Subject: [PATCH 1/2] Fix test `02789_reading_from_s3_with_connection_pool` --- ...89_reading_from_s3_with_connection_pool.sh | 84 +++++++++++-------- 1 file changed, 50 insertions(+), 34 deletions(-) diff --git a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh index 751b2798243..39399842db1 100755 --- a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh +++ b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh @@ -14,43 +14,59 @@ SETTINGS disk = 's3_disk', min_bytes_for_wide_part = 0; INSERT INTO test_s3 SELECT number, number FROM numbers_mt(1e7); " -query="SELECT a, b FROM test_s3" -query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1) -${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" -${CLICKHOUSE_CLIENT} -nm --query " -SELECT ProfileEvents['DiskConnectionsPreserved'] > 0 -FROM system.query_log -WHERE type = 'QueryFinish' - AND current_database = currentDatabase() - AND query_id='$query_id'; -" + +# This (reusing connections from the pool) is not guaranteed to always happen, +# (due to random time difference between the queries and random activity in parallel) +# but should happen most of the time. + +while true +do + query="SELECT a, b FROM test_s3" + query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1) + ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" + + RES=$(${CLICKHOUSE_CLIENT} -nm --query " + SELECT ProfileEvents['DiskConnectionsPreserved'] > 0 + FROM system.query_log + WHERE type = 'QueryFinish' + AND current_database = currentDatabase() + AND query_id='$query_id'; + ") + + [[ $RES -eq 1 ]] && echo "$RES" && break; +done # Test connection pool in ReadWriteBufferFromHTTP -query_id=$(${CLICKHOUSE_CLIENT} -nq " -create table mut (n int, m int, k int) engine=ReplicatedMergeTree('/test/02441/{database}/mut', '1') order by n; -set insert_keeper_fault_injection_probability=0; -insert into mut values (1, 2, 3), (10, 20, 30); +while true +do + query_id=$(${CLICKHOUSE_CLIENT} -nq " + create table mut (n int, m int, k int) engine=ReplicatedMergeTree('/test/02441/{database}/mut', '1') order by n; + set insert_keeper_fault_injection_probability=0; + insert into mut values (1, 2, 3), (10, 20, 30); -system stop merges mut; -alter table mut delete where n = 10; + system stop merges mut; + alter table mut delete where n = 10; -select queryID() from( - -- a funny way to wait for a MUTATE_PART to be assigned - select sleepEachRow(2) from url('http://localhost:8123/?param_tries={1..10}&query=' || encodeURLComponent( - 'select 1 where ''MUTATE_PART'' not in (select type from system.replication_queue where database=''' || currentDatabase() || ''' and table=''mut'')' - ), 'LineAsString', 's String') - -- queryID() will be returned for each row, since the query above doesn't return anything we need to return a fake row - union all - select 1 -) limit 1 settings max_threads=1; -" 2>&1) -${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" -${CLICKHOUSE_CLIENT} -nm --query " -SELECT ProfileEvents['StorageConnectionsPreserved'] > 0 -FROM system.query_log -WHERE type = 'QueryFinish' - AND current_database = currentDatabase() - AND query_id='$query_id'; -" + select queryID() from( + -- a funny way to wait for a MUTATE_PART to be assigned + select sleepEachRow(2) from url('http://localhost:8123/?param_tries={1..10}&query=' || encodeURLComponent( + 'select 1 where ''MUTATE_PART'' not in (select type from system.replication_queue where database=''' || currentDatabase() || ''' and table=''mut'')' + ), 'LineAsString', 's String') + -- queryID() will be returned for each row, since the query above doesn't return anything we need to return a fake row + union all + select 1 + ) limit 1 settings max_threads=1; + " 2>&1) + ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" + RES=$(${CLICKHOUSE_CLIENT} -nm --query " + SELECT ProfileEvents['StorageConnectionsPreserved'] > 0 + FROM system.query_log + WHERE type = 'QueryFinish' + AND current_database = currentDatabase() + AND query_id='$query_id'; + ") + + [[ $RES -eq 1 ]] && echo "$RES" && break; +done From ff0b8889ab8bf347343e18b4e9a407b3cab8264b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 5 Aug 2024 14:36:24 +0200 Subject: [PATCH 2/2] Update 02789_reading_from_s3_with_connection_pool.sh --- .../0_stateless/02789_reading_from_s3_with_connection_pool.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh index 39399842db1..5a37d51233d 100755 --- a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh +++ b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-random-settings, no-replicated-database +# Tags: no-fasttest, no-random-settings, no-replicated-database, no-distributed-cache CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh