adjust perf test thresholds

2024-11-24 00:22:29 +00:00 · 2021-03-26 00:00:18 +03:00 · 2021-03-26 00:00:18 +03:00 · a12cc5f559
commit a12cc5f559
parent f784bd4051
31 changed files with 31 additions and 31 deletions
--- a/tests/performance/arithmetic.xml
+++ b/tests/performance/arithmetic.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">
    <settings>
        <max_memory_usage>30000000000</max_memory_usage>
    </settings>
--- a/tests/performance/array_join.xml
+++ b/tests/performance/array_join.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">



--- a/tests/performance/bounding_ratio.xml
+++ b/tests/performance/bounding_ratio.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">
    <query>SELECT boundingRatio(number, number) FROM numbers(100000000)</query>
    <query>SELECT (argMax(number, number) - argMin(number, number)) / (max(number) - min(number)) FROM numbers(100000000)</query>
 </test>
--- a/tests/performance/codecs_float_insert.xml
+++ b/tests/performance/codecs_float_insert.xml
@ -1,5 +1,5 @@
 <!-- FIXME this instability is abysmal, investigate the unstable queries -->
-<test>
+<test max_ignored_relative_change="0.2">
    <settings>
        <allow_suspicious_codecs>1</allow_suspicious_codecs>
    </settings>
--- a/tests/performance/codecs_int_insert.xml
+++ b/tests/performance/codecs_int_insert.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">
    <settings>
        <allow_suspicious_codecs>1</allow_suspicious_codecs>
    </settings>
--- a/tests/performance/conditional.xml
+++ b/tests/performance/conditional.xml
@ -1,4 +1,4 @@
-<test max_ignored_relative_change="0.2">
+<test>
    <query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(if(rand() % 2, toDateTime('2019-02-04 01:24:31'), toDate('2019-02-04')))</query>
    <query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(multiIf(rand() % 2, toDateTime('2019-02-04 01:24:31'), toDate('2019-02-04')))</query>
    <query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(if(rand() % 2, [toDateTime('2019-02-04 01:24:31')], [toDate('2019-02-04')]))</query>
--- a/tests/performance/constant_column_search.xml
+++ b/tests/performance/constant_column_search.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">
    <tags>
        <tag>search</tag>
    </tags>
--- a/tests/performance/date_time_64.xml
+++ b/tests/performance/date_time_64.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">
    <preconditions>
        <table_exists>hits_100m_single</table_exists>
    </preconditions>
--- a/tests/performance/date_time_long.xml
+++ b/tests/performance/date_time_long.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.3">
    <substitutions>
       <substitution>
           <name>datetime_transform</name>
--- a/tests/performance/direct_dictionary.xml
+++ b/tests/performance/direct_dictionary.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">
    <create_query>
        CREATE TABLE simple_direct_dictionary_test_table
        (
--- a/tests/performance/float_formatting.xml
+++ b/tests/performance/float_formatting.xml
@ -3,7 +3,7 @@
    is 10 times faster than toString(number % 100 + 0.5). The shorter
    queries are somewhat unstable, so ignore differences less than 10%.
 -->
-<test max_ignored_relative_change="0.2">
+<test>
    <substitutions>
       <substitution>
           <name>expr</name>
--- a/tests/performance/fuzz_bits.xml
+++ b/tests/performance/fuzz_bits.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.3">



--- a/tests/performance/general_purpose_hashes.xml
+++ b/tests/performance/general_purpose_hashes.xml
@ -1,4 +1,4 @@
-<test max_ignored_relative_change="0.2">
+<test>
    <substitutions>
        <substitution>
           <name>gp_hash_func</name>
--- a/tests/performance/generate_table_function.xml
+++ b/tests/performance/generate_table_function.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">
    <query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') LIMIT 1000000000);</query>
    <query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 0, 10, 10) LIMIT 1000000000);</query>
    <query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)', 0, 10, 10) LIMIT 1000000000);</query>
--- a/tests/performance/group_by_sundy_li.xml
+++ b/tests/performance/group_by_sundy_li.xml
@ -1,4 +1,4 @@
-<test max_ignored_relative_change="0.2">
+<test max_ignored_relative_change="0.4">
    <settings>
        <max_insert_threads>8</max_insert_threads>
    </settings>
--- a/tests/performance/if_array_string.xml
+++ b/tests/performance/if_array_string.xml
@ -1,4 +1,4 @@
-<test max_ignored_relative_change="0.4">
+<test max_ignored_relative_change="0.2">
    <query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : ['a', 'b', 'c'])</query>
    <query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(rand() % 2 ? materialize(['Hello', 'World']) : ['a', 'b', 'c'])</query>
    <query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : materialize(['a', 'b', 'c']))</query>
--- a/tests/performance/int_parsing.xml
+++ b/tests/performance/int_parsing.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">
    <preconditions>
        <table_exists>hits_100m_single</table_exists>
        <table_exists>hits_10m_single</table_exists>
--- a/tests/performance/jit_small_requests.xml
+++ b/tests/performance/jit_small_requests.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">
    <query>
        WITH
            bitXor(number, 0x4CF2D2BAAE6DA887) AS x0,
--- a/tests/performance/joins_in_memory.xml
+++ b/tests/performance/joins_in_memory.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">
    <create_query>CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory</create_query>

    <fill_query>INSERT INTO ints SELECT number AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000)</fill_query>
--- a/tests/performance/joins_in_memory_pmj.xml
+++ b/tests/performance/joins_in_memory_pmj.xml
@ -1,4 +1,4 @@
-<test max_ignored_relative_change="1.3">
+<test max_ignored_relative_change="0.7">
    <create_query>CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory</create_query>

    <settings>
--- a/tests/performance/logical_functions_medium.xml
+++ b/tests/performance/logical_functions_medium.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">
    <settings>
        <max_threads>1</max_threads>
    </settings>
--- a/tests/performance/logical_functions_small.xml
+++ b/tests/performance/logical_functions_small.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">
    <settings>
        <max_threads>1</max_threads>
    </settings>
--- a/tests/performance/math.xml
+++ b/tests/performance/math.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.3">
    <substitutions>
        <substitution>
           <name>func_slow</name>
--- a/tests/performance/optimized_select_final.xml
+++ b/tests/performance/optimized_select_final.xml
@ -1,4 +1,4 @@
-<test max_ignored_relative_change="0.2">
+<test max_ignored_relative_change="0.3">
    <settings>
        <do_not_merge_across_partitions_select_final>1</do_not_merge_across_partitions_select_final>
    </settings>
--- a/tests/performance/optimized_select_final_one_part.xml
+++ b/tests/performance/optimized_select_final_one_part.xml
@ -1,4 +1,4 @@
-<test max_ignored_relative_change="0.2">
+<test>
    <settings>
        <do_not_merge_across_partitions_select_final>1</do_not_merge_across_partitions_select_final>
    </settings>
--- a/tests/performance/or_null_default.xml
+++ b/tests/performance/or_null_default.xml
@ -1,4 +1,4 @@
-<test max_ignored_relative_change="0.3">
+<test>
    <query>SELECT sumOrNull(number) FROM numbers(100000000)</query>
    <query>SELECT sumOrDefault(toNullable(number)) FROM numbers(100000000)</query>
    <query>SELECT sumOrNull(number) FROM numbers(10000000) GROUP BY number % 1024</query>
--- a/tests/performance/parse_engine_file.xml
+++ b/tests/performance/parse_engine_file.xml
@ -1,4 +1,4 @@
-<test max_ignored_relative_change="0.4">
+<test max_ignored_relative_change="0.2">

 <preconditions>
    <table_exists>test.hits</table_exists>
--- a/tests/performance/random_string.xml
+++ b/tests/performance/random_string.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">
    <query>SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomString(10))</query>
    <query>SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomString(100))</query>
    <query>SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomString(1000))</query>
--- a/tests/performance/sum.xml
+++ b/tests/performance/sum.xml
@ -1,4 +1,4 @@
-<test max_ignored_relative_change="0.2">
+<test>
    <query>SELECT sum(number) FROM numbers(100000000)</query>
    <query>SELECT sum(toUInt32(number)) FROM numbers(100000000)</query>
    <query>SELECT sum(toUInt16(number)) FROM numbers(100000000)</query>
--- a/tests/performance/sum_map.xml
+++ b/tests/performance/sum_map.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.3">
    <settings>
        <max_threads>1</max_threads>
    </settings>
--- a/tests/performance/url_hits.xml
+++ b/tests/performance/url_hits.xml
@ -1,4 +1,4 @@
-<test>
+<test max_ignored_relative_change="0.2">
    <preconditions>
        <table_exists>hits_100m_single</table_exists>
        <table_exists>hits_10m_single</table_exists>