Adjust ignore thresholds for unstable perf tests
Based on historical data.
```
SELECT
test,
ceil(max(q[3]), 1) AS h
FROM
(
SELECT
test,
query_index,
count(*),
min(event_time),
max(event_time) AS t,
arrayMap(x -> floor(x, 3), quantiles(0, 0.5, 0.95, 1)(stat_threshold)) AS q,
median(stat_threshold) AS m
FROM perftest.query_metrics
WHERE (metric = 'client_time') AND (abs(diff) < 0.05)
GROUP BY
test,
query_index,
query_display_name
HAVING (t > '2020-09-01 00:00:00') AND (m > 0.1)
ORDER BY m DESC
)
GROUP BY test
ORDER BY h DESC
FORMAT TSV
cryptographic_hashes 1.3
collations 0.8
joins_in_memory_pmj 0.8
joins_in_memory 0.7
merge_tree_simple_select 0.7
set_index 0.7
decimal_casts 0.7
website 0.6
logical_functions_medium 0.5
count 0.5
merge_tree_many_partitions 0.5
decimal_aggregates 0.5
codecs_int_insert 0.5
column_column_comparison 0.5
insert_parallel 0.4
parse_engine_file 0.4
read_in_order_many_parts 0.4
logical_functions_small 0.4
parallel_insert 0.3
parallel_index 0.3
push_down_limit 0.3
jit_large_requests 0.3
select_format 0.3
arithmetic 0.3
merge_tree_huge_pk 0.3
materialized_view_parallel_insert 0.3
columns_hashing 0.3
if_array_string 0.3
random_string 0.2
random_printable_ascii 0.2
set 0.2
empty_string_serialization 0.2
```
To apply:
```
sed 's/^\(.*\) \(.*\)$/sed -i "s\/^<test.*$\/<test max_ignored_relative_change="'"'"\2">\/g" tests\/performance\/\1.xml/g' ../bad.tsv | bash
```
2020-09-16 15:27:51 +00:00
|
|
|
<test max_ignored_relative_change="0.5">
|
2020-07-09 10:46:16 +00:00
|
|
|
<settings>
|
2020-07-31 16:28:04 +00:00
|
|
|
<max_memory_usage>35G</max_memory_usage>
|
2020-07-09 10:46:16 +00:00
|
|
|
</settings>
|
|
|
|
|
|
2019-12-12 13:11:07 +00:00
|
|
|
<create_query>CREATE TABLE t (x UInt64, d32 Decimal32(3), d64 Decimal64(4), d128 Decimal128(5)) ENGINE = Memory</create_query>
|
2020-07-11 10:35:44 +00:00
|
|
|
<!-- use less threads to save memory -->
|
2020-07-13 11:40:46 +00:00
|
|
|
<fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(500000000) SETTINGS max_threads = 8</fill_query>
|
2019-12-12 13:11:07 +00:00
|
|
|
<drop_query>DROP TABLE IF EXISTS t</drop_query>
|
|
|
|
|
|
|
|
|
|
<query>SELECT min(d32), max(d32), argMin(x, d32), argMax(x, d32) FROM t</query>
|
|
|
|
|
<query>SELECT min(d64), max(d64), argMin(x, d64), argMax(x, d64) FROM t</query>
|
|
|
|
|
<query>SELECT min(d128), max(d128), argMin(x, d128), argMax(x, d128) FROM t</query>
|
2021-02-11 13:37:46 +00:00
|
|
|
|
2019-12-12 13:11:07 +00:00
|
|
|
<query>SELECT avg(d32), sum(d32), sumWithOverflow(d32) FROM t</query>
|
|
|
|
|
<query>SELECT avg(d64), sum(d64), sumWithOverflow(d64) FROM t</query>
|
|
|
|
|
<query>SELECT avg(d128), sum(d128), sumWithOverflow(d128) FROM t</query>
|
|
|
|
|
|
2020-07-14 12:55:25 +00:00
|
|
|
<query>SELECT uniq(d32), uniqCombined(d32), uniqExact(d32), uniqHLL12(d32) FROM (SELECT * FROM t LIMIT 10000000)</query>
|
|
|
|
|
<query>SELECT uniq(d64), uniqCombined(d64), uniqExact(d64), uniqHLL12(d64) FROM (SELECT * FROM t LIMIT 10000000)</query>
|
|
|
|
|
<query>SELECT uniq(d128), uniqCombined(d128), uniqExact(d128), uniqHLL12(d128) FROM (SELECT * FROM t LIMIT 1000000)</query>
|
2021-02-11 13:37:46 +00:00
|
|
|
|
2020-07-14 12:55:25 +00:00
|
|
|
<query>SELECT median(d32), medianExact(d32), medianExactWeighted(d32, 2) FROM (SELECT * FROM t LIMIT 10000000)</query>
|
|
|
|
|
<query>SELECT median(d64), medianExact(d64), medianExactWeighted(d64, 2) FROM (SELECT * FROM t LIMIT 1000000)</query>
|
|
|
|
|
<query>SELECT median(d128), medianExact(d128), medianExactWeighted(d128, 2) FROM (SELECT * FROM t LIMIT 1000000)</query>
|
2021-02-11 13:37:46 +00:00
|
|
|
|
2020-07-14 12:55:25 +00:00
|
|
|
<query>SELECT quantile(d32), quantileExact(d32), quantileExactWeighted(d32, 2) FROM (SELECT * FROM t LIMIT 10000000)</query>
|
|
|
|
|
<query>SELECT quantile(d64), quantileExact(d64), quantileExactWeighted(d64, 2) FROM (SELECT * FROM t LIMIT 1000000)</query>
|
|
|
|
|
<query>SELECT quantile(d128), quantileExact(d128), quantileExactWeighted(d128, 2) FROM (SELECT * FROM t LIMIT 1000000)</query>
|
2019-12-12 13:11:07 +00:00
|
|
|
|
2020-07-14 12:55:25 +00:00
|
|
|
<query>SELECT quantilesExact(0.1, 0.9)(d32), quantilesExactWeighted(0.1, 0.9)(d32, 2) FROM (SELECT * FROM t LIMIT 10000000)</query>
|
|
|
|
|
<query>SELECT quantilesExact(0.1, 0.9)(d64), quantilesExactWeighted(0.1, 0.9)(d64, 2) FROM (SELECT * FROM t LIMIT 1000000)</query>
|
|
|
|
|
<query>SELECT quantilesExact(0.1, 0.9)(d128), quantilesExactWeighted(0.1, 0.9)(d128, 2) FROM (SELECT * FROM t LIMIT 1000000)</query>
|
2021-02-11 13:37:46 +00:00
|
|
|
|
2019-12-12 13:11:07 +00:00
|
|
|
<query>SELECT varPop(d32), varSamp(d32), stddevPop(d32) FROM t</query>
|
2021-02-11 13:37:46 +00:00
|
|
|
<query>SELECT varPop(d64), varSamp(d64), stddevPop(d64) FROM (SELECT * FROM t LIMIT 10000000)</query>
|
|
|
|
|
<query>SELECT varPop(d128), varSamp(d128), stddevPop(d128) FROM (SELECT * FROM t LIMIT 10000000)</query>
|
2019-12-12 13:11:07 +00:00
|
|
|
</test>
|