thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adjust ignore thresholds for unstable perf tests

Browse Source 
Based on historical data.

```
SELECT
    test,
    ceil(max(q[3]), 1) AS h
FROM
(
    SELECT
        test,
        query_index,
        count(*),
        min(event_time),
        max(event_time) AS t,
        arrayMap(x -> floor(x, 3), quantiles(0, 0.5, 0.95, 1)(stat_threshold)) AS q,
        median(stat_threshold) AS m
    FROM perftest.query_metrics
    WHERE (metric = 'client_time') AND (abs(diff) < 0.05)
    GROUP BY
        test,
        query_index,
        query_display_name
    HAVING (t > '2020-09-01 00:00:00') AND (m > 0.1)
    ORDER BY m DESC
)
GROUP BY test
ORDER BY h DESC
FORMAT TSV

cryptographic_hashes	1.3
collations	0.8
joins_in_memory_pmj	0.8
joins_in_memory	0.7
merge_tree_simple_select	0.7
set_index	0.7
decimal_casts	0.7
website	0.6
logical_functions_medium	0.5
count	0.5
merge_tree_many_partitions	0.5
decimal_aggregates	0.5
codecs_int_insert	0.5
column_column_comparison	0.5
insert_parallel	0.4
parse_engine_file	0.4
read_in_order_many_parts	0.4
logical_functions_small	0.4
parallel_insert	0.3
parallel_index	0.3
push_down_limit	0.3
jit_large_requests	0.3
select_format	0.3
arithmetic	0.3
merge_tree_huge_pk	0.3
materialized_view_parallel_insert	0.3
columns_hashing	0.3
if_array_string	0.3
random_string	0.2
random_printable_ascii	0.2
set	0.2
empty_string_serialization	0.2
```

To apply:
```
sed 's/^\(.*\)        \(.*\)$/sed -i "s\/^<test.*$\/<test max_ignored_relative_change="'"'"\2">\/g" tests\/performance\/\1.xml/g' ../bad.tsv | bash
```
This commit is contained in:
Alexander Kuzmenkov 2020-09-16 18:27:51 +03:00
parent faa5190f11
commit 0f8aec59a3
28 changed files with 28 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/performance/codecs_int_insert.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test max_ignored_relative_change="0.5">
<settings>
<allow_suspicious_codecs>1</allow_suspicious_codecs>
</settings>

2
tests/performance/collations.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.8">

2
tests/performance/column_column_comparison.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.5">
<tags>
<tag>comparison</tag>
</tags>

2
tests/performance/columns_hashing.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.3">
<tags>
<tag>columns_hashing</tag>
</tags>

2
tests/performance/count.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.5">
<create_query>CREATE TABLE data(k UInt64, v UInt64) ENGINE = MergeTree ORDER BY k</create_query>
<fill_query>INSERT INTO data SELECT number, 1 from numbers(10000000)</fill_query>

2
tests/performance/cryptographic_hashes.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="1.0">
<test max_ignored_relative_change="1.3">
<substitutions>
<substitution>
<name>hash_slow</name>

2
tests/performance/decimal_aggregates.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.4">
<test max_ignored_relative_change="0.5">
<settings>
<max_memory_usage>35G</max_memory_usage>
</settings>

2
tests/performance/empty_string_serialization.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.2">
<!-- gcc-8 generates 20% faster code than gcc-9

2
tests/performance/if_array_string.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test max_ignored_relative_change="0.3">
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : ['a', 'b', 'c'])</query>
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(rand() % 2 ? materialize(['Hello', 'World']) : ['a', 'b', 'c'])</query>
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : materialize(['a', 'b', 'c']))</query>

2
tests/performance/insert_parallel.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.4">
<settings>
<max_insert_threads>4</max_insert_threads>
</settings>

2
tests/performance/jit_large_requests.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.3">
<create_query>
CREATE TABLE jit_test (
a UInt64,

2
tests/performance/joins_in_memory.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test max_ignored_relative_change="0.7">
<create_query>CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory</create_query>
<fill_query>INSERT INTO ints SELECT number AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000)</fill_query>

2
tests/performance/logical_functions_medium.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.4">
<test max_ignored_relative_change="0.5">
<settings>
<max_threads>1</max_threads>
</settings>

2
tests/performance/materialized_view_parallel_insert.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.3">
<preconditions>
<table_exists>hits_10m_single</table_exists>
</preconditions>

2
tests/performance/merge_tree_huge_pk.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.3">
<create_query>
CREATE TABLE huge_pk ENGINE = MergeTree ORDER BY (
c001, c002, c003, c004, c005, c006, c007, c008, c009, c010, c011, c012, c013, c014, c015, c016, c017, c018, c019, c020,

2
tests/performance/merge_tree_many_partitions.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.5">
<create_query>CREATE TABLE bad_partitions (x UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x</create_query>
<fill_query>INSERT INTO bad_partitions SELECT * FROM numbers(10000)</fill_query>

2
tests/performance/merge_tree_simple_select.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.7">
<create_query>CREATE TABLE simple_mergetree (EventDate Date, x UInt64) ENGINE = MergeTree ORDER BY x</create_query>
<fill_query>INSERT INTO simple_mergetree SELECT number, today() + intDiv(number, 10000000) FROM numbers_mt(100000000)</fill_query>
<fill_query>OPTIMIZE TABLE simple_mergetree FINAL</fill_query>

2
tests/performance/parallel_index.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.3">
<create_query>create table test_parallel_index (x UInt64, y UInt64, z UInt64, INDEX a (y) TYPE minmax GRANULARITY 2,
INDEX b (z) TYPE set(8) GRANULARITY 2) engine = MergeTree order by x partition by bitAnd(x, 63 * 64) settings index_granularity = 4;</create_query>

2
tests/performance/parallel_insert.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.3">
<preconditions>
<table_exists>hits_10m_single</table_exists>

2
tests/performance/parse_engine_file.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test max_ignored_relative_change="0.4">
<create_query>CREATE TABLE IF NOT EXISTS table_{format} ENGINE = File({format}) AS test.hits</create_query>

2
tests/performance/push_down_limit.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.3">
<query>select number from (select number from numbers(10000000) order by -number) limit 10</query>
<query>select number from (select number from numbers_mt(100000000) order by -number) limit 10</query>
</test>

2
tests/performance/random_printable_ascii.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.2">
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomPrintableASCII(10))</query>
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomPrintableASCII(100))</query>
<query>SELECT count() FROM zeros(100000) WHERE NOT ignore(randomPrintableASCII(1000))</query>

2
tests/performance/random_string.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.2">
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomString(10))</query>
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomString(100))</query>
<query>SELECT count() FROM zeros(100000) WHERE NOT ignore(randomString(1000))</query>

2
tests/performance/read_in_order_many_parts.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.4">
<settings>
<optimize_aggregation_in_order>1</optimize_aggregation_in_order>
<optimize_read_in_order>1</optimize_read_in_order>

2
tests/performance/select_format.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test max_ignored_relative_change="0.3">
<settings>
<output_format_pretty_max_rows>1000000</output_format_pretty_max_rows>
<max_threads>1</max_threads>

2
tests/performance/set.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.2">
<substitutions>
<substitution>
<name>table_small</name>

2
tests/performance/set_index.xml
View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.7">
<create_query>CREATE TABLE test_in (`a` UInt32) ENGINE = MergeTree() ORDER BY a</create_query>
<fill_query>INSERT INTO test_in SELECT number FROM numbers(500000000)</fill_query>

2
tests/performance/website.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test max_ignored_relative_change="0.6">
<preconditions>
<table_exists>hits_10m_single</table_exists>