thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

better 01193_metadata_loading

Browse Source
This commit is contained in:
Alexander Tokmakov 2020-09-17 18:54:41 +03:00
parent a76d8fb96b
commit 06ff6d2eda
2 changed files with 20 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tests/queries/0_stateless/01193_metadata_loading.reference
View File

@ -1,5 +1,5 @@
10000 0 2020-06-25 hello [1,2] [3,4] 1000 0 2020-06-25 hello [1,2] [3,4]
10000 1 2020-06-26 word [10,20] [30,40] 1000 1 2020-06-26 word [10,20] [30,40]
ok ok
8000 0 2020-06-25 hello [1,2] [3,4] 8000 0 2020-06-25 hello [1,2] [3,4]
8000 1 2020-06-26 word [10,20] [30,40] 8000 1 2020-06-26 word [10,20] [30,40]

36
tests/queries/0_stateless/01193_metadata_loading.sh
View File

@ -6,14 +6,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# it is the worst way of making performance test, nevertheless it can detect significant slowdown and some other issues, that usually found by stress test # it is the worst way of making performance test, nevertheless it can detect significant slowdown and some other issues, that usually found by stress test
db="test_01193_$RANDOM" db="test_01193_$RANDOM"
declare -A engines
engines[0]="Memory"
engines[1]="File(CSV)"
engines[2]="Log"
engines[3]="StripeLog"
engines[4]="MergeTree ORDER BY i"
tables=1000 tables=1000
threads=10 threads=10
count_multiplier=1 count_multiplier=1
@ -24,11 +16,17 @@ debug_or_sanitizer_build=$($CLICKHOUSE_CLIENT -q "WITH ((SELECT value FROM syste
if [[ debug_or_sanitizer_build -eq 1 ]]; then tables=100; count_multiplier=10; max_time_ms=1500; fi if [[ debug_or_sanitizer_build -eq 1 ]]; then tables=100; count_multiplier=10; max_time_ms=1500; fi
create_tables() { create_tables() {
for i in $(seq 1 $tables); do $CLICKHOUSE_CLIENT -q "WITH
engine=${engines[$((i % ${#engines[@]}))]} 'CREATE TABLE $db.table_$1_' AS create1,
$CLICKHOUSE_CLIENT -q "CREATE TABLE $db.table_$1_$i (i UInt64, d Date, s String, n Nested(i UInt8, f Float32)) ENGINE=$engine" ' (i UInt64, d Date, s String, n Nested(i UInt8, f Float32)) ENGINE=' AS create2,
$CLICKHOUSE_CLIENT -q "INSERT INTO $db.table_$1_$i VALUES (0, '2020-06-25', 'hello', [1, 2], [3, 4]), (1, '2020-06-26', 'word', [10, 20], [30, 40])" ['Memory', 'File(CSV)', 'Log', 'StripeLog', 'MergeTree ORDER BY i'] AS engines,
done 'INSERT INTO $db.table_$1_' AS insert1,
' VALUES (0, ''2020-06-25'', ''hello'', [1, 2], [3, 4]), (1, ''2020-06-26'', ''word'', [10, 20], [30, 40])' AS insert2
SELECT arrayStringConcat(
groupArray(
create1 || toString(number) || create2 || engines[1 + number % length(engines)] || ';\n' ||
insert1 || toString(number) || insert2
), ';\n') FROM numbers($tables) FORMAT TSVRaw;" | $CLICKHOUSE_CLIENT -nm
} }
$CLICKHOUSE_CLIENT -q "CREATE DATABASE $db" $CLICKHOUSE_CLIENT -q "CREATE DATABASE $db"
@ -39,15 +37,15 @@ done
wait wait
$CLICKHOUSE_CLIENT -q "CREATE TABLE $db.table_merge (i UInt64, d Date, s String, n Nested(i UInt8, f Float32)) ENGINE=Merge('$db', '^table_')" $CLICKHOUSE_CLIENT -q "CREATE TABLE $db.table_merge (i UInt64, d Date, s String, n Nested(i UInt8, f Float32)) ENGINE=Merge('$db', '^table_')"
$CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM $db.table_merge GROUP BY i, d, s, n.i, n.f ORDER BY i" $CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM merge('$db', '^table_9') GROUP BY i, d, s, n.i, n.f ORDER BY i"
for i in {1..5}; do
$CLICKHOUSE_CLIENT -q "DETACH DATABASE $db" $CLICKHOUSE_CLIENT -q "DETACH DATABASE $db"
$CLICKHOUSE_CLIENT -q "ATTACH DATABASE $db" --query_id="$db-$i";
done
# get real time, grep seconds, remove point, remove leading zeros $CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS"
elapsed_ms=$({ time $CLICKHOUSE_CLIENT -q "ATTACH DATABASE $db"; } 2>&1 | grep real | grep -Po "0m\K[0-9\.]*" | tr -d '.' | sed "s/^0*//") $CLICKHOUSE_CLIENT -q "SELECT if(quantile(0.5)(query_duration_ms) < $max_time_ms, 'ok', toString(groupArray(query_duration_ms))) FROM system.query_log WHERE query_id LIKE '$db-%' AND type=2"
$CLICKHOUSE_CLIENT -q "SELECT '01193_metadata_loading', $elapsed_ms FORMAT Null" # it will be printed to server log
if [[ $elapsed_ms -le $max_time_ms ]]; then echo ok; fi
$CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM $db.table_merge GROUP BY i, d, s, n.i, n.f ORDER BY i" $CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM $db.table_merge GROUP BY i, d, s, n.i, n.f ORDER BY i"