Merge pull request #16556 from ClickHouse/aku/perf-after-numa

Adjust perf test thresholds after NUMA binding
This commit is contained in:
Alexander Kuzmenkov 2020-11-02 09:40:11 +03:00 committed by GitHub
commit dc5fc9ddcb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
64 changed files with 90 additions and 124 deletions

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3">
<test>
<preconditions>
<table_exists>hits_100m_single</table_exists>
</preconditions>

View File

@ -1,19 +1,6 @@
<test>
<stop_conditions>
<all_of>
<iterations>10</iterations>
</all_of>
</stop_conditions>
<query>SELECT any(-1 * (((-2 * (number * -3)) * -4) * -5)) FROM numbers(120000000)</query>
<query>SELECT anyLast(-1 * (((-2 * (number * -3)) * -4) * -5)) FROM numbers(120000000)</query>
<query>SELECT any(number * 2) as n, n * 3 FROM numbers(120000000)</query>
<query>SELECT any(number * round(toInt64(number), -2)) FROM numbers(120000000)</query>
<query>SELECT any(-1 * (((-2 * (number * -3)) * -4) * -5)) FROM numbers(500000000)</query>
<query>SELECT anyLast(-1 * (((-2 * (number * -3)) * -4) * -5)) FROM numbers(500000000)</query>
<query>SELECT any(number * 2) as n, n * 3 FROM numbers(500000000)</query>
<query>SELECT any(number * round(toInt64(number), -2)) FROM numbers(500000000)</query>
</test>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.4">
<test>
<settings>
<max_memory_usage>30000000000</max_memory_usage>
</settings>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test>
<create_query>DROP TABLE IF EXISTS perf_lc_str</create_query>
<create_query>CREATE TABLE perf_lc_str(
str LowCardinality(String),

View File

@ -1,5 +1,5 @@
<!-- FIXME this instability is abysmal, investigate the unstable queries -->
<test max_ignored_relative_change="1.5">
<test>
<settings>
<allow_suspicious_codecs>1</allow_suspicious_codecs>
</settings>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test>
<settings>
<allow_suspicious_codecs>1</allow_suspicious_codecs>
</settings>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.8">
<test>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.5">
<test>
<tags>
<tag>comparison</tag>
</tags>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3">
<test>
<preconditions>
<table_exists>hits_10m_single</table_exists>
<table_exists>hits_100m_single</table_exists>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.5">
<test>
<create_query>CREATE TABLE data(k UInt64, v UInt64) ENGINE = MergeTree ORDER BY k</create_query>
<fill_query>INSERT INTO data SELECT number, 1 from numbers(10000000)</fill_query>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3">
<test>
<preconditions>
<table_exists>hits_100m_single</table_exists>
<table_exists>hits_10m_single</table_exists>
@ -67,7 +67,7 @@ PageCharset тоже почти всегда непуст, но его сред
<query>SELECT uniqCombined(17)(UserID) FROM hits_100m_single</query>
<!-- 44. Разные алгоритмы вычисления кардинальности. -->
<query>SELECT uniqExact(UserID) FROM hits_10m_single SETTINGS max_threads = 1</query>
<query>SELECT uniqExact(UserID) FROM hits_10m_single</query>
<query>SELECT uniqExact(UserID) FROM hits_100m_single</query>
<!-- 45. Что-то чуть более похожее на реальный запрос. -->
<query>SELECT RegionID, uniq(UserID) FROM hits_10m_single GROUP BY RegionID SETTINGS max_threads = 1</query>
<query>SELECT RegionID, uniq(UserID) FROM hits_100m_single GROUP BY RegionID</query>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="1.0">
<test>
<substitutions>
<substitution>
<name>hash_slow</name>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3">
<test>
<preconditions>
<table_exists>hits_100m_single</table_exists>
</preconditions>

View File

@ -1,11 +1,11 @@
<test max_ignored_relative_change="0.15">
<test max_ignored_relative_change="0.2">
<settings>
<max_memory_usage>15G</max_memory_usage>
</settings>
<create_query>CREATE TABLE t (x UInt64, d32 Decimal32(3), d64 Decimal64(4), d128 Decimal128(5)) ENGINE = Memory</create_query>
<!-- use less threads to save memory -->
<fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(100000000) SETTINGS max_threads = 8</fill_query>
<fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(200000000) SETTINGS max_threads = 8</fill_query>
<drop_query>DROP TABLE IF EXISTS t</drop_query>
<query>SELECT toUInt32(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t FORMAT Null</query>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test>
<!-- gcc-8 generates 20% faster code than gcc-9

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3">
<test>
<preconditions>
<table_exists>hits_100m_single</table_exists>
<table_exists>hits_10m_single</table_exists>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.4">
<test>
<preconditions>
<table_exists>test.hits</table_exists>
</preconditions>

View File

@ -3,7 +3,7 @@
is 10 times faster than toString(number % 100 + 0.5). The shorter
queries are somewhat unstable, so ignore differences less than 10%.
-->
<test max_ignored_relative_difference="0.1">
<test>
<substitutions>
<substitution>
<name>expr</name>

View File

@ -1,5 +1,5 @@
<test>
<query>SELECT count() FROM numbers(1000000) WHERE NOT ignore(formatReadableSize(number))</query>
<query>SELECT count() FROM numbers(1000000) WHERE NOT ignore(formatReadableQuantity(number))</query>
<query>SELECT count() FROM numbers(1000000) WHERE NOT ignore(formatReadableTimeDelta(number))</query>
<query>SELECT count() FROM numbers(10000000) WHERE NOT ignore(formatReadableSize(number))</query>
<query>SELECT count() FROM numbers(10000000) WHERE NOT ignore(formatReadableQuantity(number))</query>
<query>SELECT count() FROM numbers(10000000) WHERE NOT ignore(formatReadableTimeDelta(number))</query>
</test>

View File

@ -1,8 +1,8 @@
<test>
<test max_ignored_relative_change="0.2">
<query>select arrayUniq(range(1 + (number % 100) * 10)) from numbers(100000) format Null</query>
<query>select arrayDistinct(range(1 + (number % 100) * 10)) from numbers(100000) format Null</query>
<query>select arrayEnumerateUniq(range(1 + (number % 100) * 10)) from numbers(100000) format Null</query>
<query>select arrayIntersect(range((1 + number % 100)), range(1, (1 + number % 100) + 1)) from numbers(100000) format Null</query>
<query>select arrayIntersect(range((1 + number % 100)), range(1, (1 + number % 100) + 1)) from numbers(1000000) format Null</query>
<query>select groupUniqArray(rand() % 100) from numbers(1000 * 1000) group by number / 1000 format Null</query>
<query>select entropy(number / 10) from numbers(1000 * 1000) group by number / 1000 format Null</query>
</test>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3">
<test>
<substitutions>
<substitution>
<name>gp_hash_func</name>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test>
<substitutions>
<substitution>
<name>hash_func</name>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test>
<!-- lon [-180; 180], lat [-90; 90] -->
<query>SELECT count() FROM numbers(1000000) WHERE NOT ignore(greatCircleDistance((rand(1) % 360) * 1. - 180, (number % 150) * 1.2 - 90, (number % 360) + toFloat64(rand(2)) / 4294967296 - 180, (rand(3) % 180) * 1. - 90))</query>
<!-- 55.755830, 37.617780 is center of Moscow -->

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.4">
<test>
<settings>
<max_insert_threads>4</max_insert_threads>
</settings>

View File

@ -61,7 +61,7 @@
intDiv( number, 4 ) as rowid,
groupArray( base64Encode( left( reinterpretAsString( rand64() ), 6) ) ) as arr
FROM
numbers(1000000)
numbers(2000000)
GROUP BY rowid
) GROUP BY id</fill_query>
<fill_query>optimize table lot_of_string_arrays_src</fill_query>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3">
<test>
<create_query>
CREATE TABLE jit_test (
a UInt64,
@ -29,7 +29,7 @@
number
FROM
system.numbers
LIMIT 10000000
LIMIT 100000000
</fill_query>
<query>
SELECT

View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.2">
<query>
WITH
bitXor(number, 0x4CF2D2BAAE6DA887) AS x0,

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.7">
<test>
<create_query>CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory</create_query>
<fill_query>INSERT INTO ints SELECT number AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000)</fill_query>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="1.0">
<test max_ignored_relative_change="0.8">
<create_query>CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory</create_query>
<create_query>SET join_algorithm = 'partial_merge'</create_query>

View File

@ -1,3 +1,3 @@
<test max_ignored_relative_change="0.3">
<test>
<query>select sum(number) from remote('127.0.0.{{1|2}}', numbers_mt(1000000000)) group by bitAnd(number, 1)</query>
</test>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.4">
<test>
<settings>
<max_threads>1</max_threads>
</settings>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.5">
<test>
<settings>
<max_threads>1</max_threads>
</settings>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.4">
<test>
<settings>
<max_threads>1</max_threads>
</settings>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3">
<test>
<preconditions>
<table_exists>hits_10m_single</table_exists>
</preconditions>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test>
<substitutions>
<substitution>
<name>func_slow</name>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3">
<test>
<create_query>
CREATE TABLE huge_pk ENGINE = MergeTree ORDER BY (
c001, c002, c003, c004, c005, c006, c007, c008, c009, c010, c011, c012, c013, c014, c015, c016, c017, c018, c019, c020,

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.5">
<test>
<create_query>CREATE TABLE bad_partitions (x UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x</create_query>
<fill_query>INSERT INTO bad_partitions SELECT * FROM numbers(10000)</fill_query>

View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.2">
<create_query>CREATE TABLE bad_partitions (a UInt64, b UInt64, c UInt64, d UInt64, e UInt64, f UInt64, g UInt64, h UInt64, i UInt64, j UInt64, k UInt64, l UInt64, m UInt64, n UInt64, o UInt64, p UInt64, q UInt64, r UInt64, s UInt64, t UInt64, u UInt64, v UInt64, w UInt64, x UInt64, y UInt64, z UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x</create_query>
<fill_query>INSERT INTO bad_partitions (x) SELECT * FROM numbers_mt(3000)</fill_query>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.7">
<test>
<create_query>CREATE TABLE simple_mergetree (EventDate Date, x UInt64) ENGINE = MergeTree ORDER BY x</create_query>
<fill_query>INSERT INTO simple_mergetree SELECT number, today() + intDiv(number, 10000000) FROM numbers_mt(100000000)</fill_query>
<fill_query>OPTIMIZE TABLE simple_mergetree FINAL</fill_query>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.4">
<test>
<settings>
<max_threads>1</max_threads>
</settings>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.4">
<test>
<substitutions>
<substitution>
<name>format</name>

View File

@ -1,13 +1,9 @@
<test>
<preconditions>
<table_exists>hits_100m_single</table_exists>
</preconditions>
<query>SELECT * FROM hits_100m_single ORDER BY CounterID, EventDate LIMIT 100</query>
<query>SELECT * FROM hits_100m_single ORDER BY CounterID, EventDate LIMIT 1000</query>
<query>SELECT * FROM hits_100m_single ORDER BY CounterID DESC, toStartOfWeek(EventDate) DESC LIMIT 100</query>
<query>SELECT * FROM hits_100m_single ORDER BY CounterID, EventDate, URL LIMIT 100</query>
<query>SELECT * FROM hits_100m_single WHERE CounterID IN (152220, 168777, 149234, 149234) ORDER BY CounterID DESC, EventDate DESC LIMIT 100</query>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.5">
<test>
<tags>
<tag>sorting</tag>
<tag>comparison</tag>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3">
<test>
<preconditions>
<table_exists>hits_10m_single</table_exists>

View File

@ -13,8 +13,7 @@
<create_query>create materialized view mv_4 engine = MergeTree order by tuple() as
select number, toString(number) from main_table where number % 13 != 4;</create_query>
<!--<query>insert into main_table select number from numbers(100000)</query>-->
<query>insert into main_table select number from numbers(1000000)</query>
<query>insert into main_table select number from numbers(10000000)</query>
<drop_query>drop table if exists main_table;</drop_query>
<drop_query>drop table if exists mv_1;</drop_query>

View File

@ -2,7 +2,7 @@
<create_query>CREATE TABLE IF NOT EXISTS table_{format} ENGINE = File({format}) AS test.hits</create_query>
<fill_query>INSERT INTO table_{format} SELECT * FROM test.hits LIMIT 100000</fill_query>
<fill_query>INSERT INTO table_{format} SELECT * FROM test.hits LIMIT 200000</fill_query>
<substitutions>
<substitution>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3">
<test>
<query>select number from (select number from numbers(10000000) order by -number) limit 10</query>
<query>select number from (select number from numbers_mt(100000000) order by -number) limit 10</query>
</test>

View File

@ -1,19 +1,9 @@
<test>
<stop_conditions>
<all_of>
<total_time_ms>10000</total_time_ms>
</all_of>
<any_of>
<total_time_ms>20000</total_time_ms>
</any_of>
</stop_conditions>
<substitutions>
<substitution>
<name>table</name>
<values>
<value>zeros(100000000)</value>
<value>zeros(1000000000)</value>
</values>
</substitution>
</substitutions>

View File

@ -1,9 +1,9 @@
<test max_ignored_relative_change="0.2">
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomString(10))</query>
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomString(100))</query>
<query>SELECT count() FROM zeros(100000) WHERE NOT ignore(randomString(1000))</query>
<query>SELECT count() FROM zeros(10000) WHERE NOT ignore(randomString(10000))</query>
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomString(rand() % 10))</query>
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomString(rand() % 100))</query>
<query>SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomString(rand() % 1000))</query>
<test>
<query>SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomString(10))</query>
<query>SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomString(100))</query>
<query>SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomString(1000))</query>
<query>SELECT count() FROM zeros(100000) WHERE NOT ignore(randomString(10000))</query>
<query>SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomString(rand() % 10))</query>
<query>SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomString(rand() % 100))</query>
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomString(rand() % 1000))</query>
</test>

View File

@ -1,12 +1,9 @@
<test>
<query>SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(10))</query>
<query>SELECT count() FROM zeros(100000) WHERE NOT ignore(randomStringUTF8(100))</query>
<query>SELECT count() FROM zeros(10000) WHERE NOT ignore(randomStringUTF8(1000))</query>
<query>SELECT count() FROM zeros(1000) WHERE NOT ignore(randomStringUTF8(10000))</query>
<query>SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(rand() % 10))</query>
<query>SELECT count() FROM zeros(100000) WHERE NOT ignore(randomStringUTF8(rand() % 100))</query>
<query>SELECT count() FROM zeros(10000) WHERE NOT ignore(randomStringUTF8(rand() % 1000))</query>
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomStringUTF8(10))</query>
<query>SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(100))</query>
<query>SELECT count() FROM zeros(100000) WHERE NOT ignore(randomStringUTF8(1000))</query>
<query>SELECT count() FROM zeros(10000) WHERE NOT ignore(randomStringUTF8(10000))</query>
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomStringUTF8(rand() % 10))</query>
<query>SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(rand() % 100))</query>
<query>SELECT count() FROM zeros(100000) WHERE NOT ignore(randomStringUTF8(rand() % 1000))</query>
</test>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test>
<query>SELECT range(number % 100) FROM numbers(10000000) FORMAT Null</query>
<query>SELECT range(0, number % 100, 1) FROM numbers(10000000) FORMAT Null</query>
</test>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.4">
<test>
<settings>
<optimize_aggregation_in_order>1</optimize_aggregation_in_order>
<optimize_read_in_order>1</optimize_read_in_order>

View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.3">
<preconditions>
<table_exists>hits_100m_single</table_exists>
</preconditions>

View File

@ -1,4 +1,4 @@
<test>
<test>
<preconditions>
<table_exists>hits_10m_single</table_exists>
<table_exists>hits_100m_single</table_exists>

View File

@ -1,10 +1,7 @@
<test>
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(roundDuration(rand() % 65536))</query>
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(roundDown(rand() % 65536, [0, 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000]))</query>
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(roundAge(rand() % 100))</query>
<query>SELECT count() FROM zeros(10000000) WHERE NOT ignore(roundDown(rand() % 100, [0, 1, 18, 25, 35, 45, 55]))</query>
<query>SELECT count() FROM numbers(10000000) WHERE NOT ignore(roundDown(rand() % 65536, (SELECT groupArray(number) FROM numbers(65536))))</query>
<query>SELECT count() FROM zeros (100000000) WHERE NOT ignore(roundDuration(rand() % 65536))</query>
<query>SELECT count() FROM zeros (100000000) WHERE NOT ignore(roundDown(rand() % 65536, [0, 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000]))</query>
<query>SELECT count() FROM zeros (100000000) WHERE NOT ignore(roundAge(rand() % 100))</query>
<query>SELECT count() FROM zeros (100000000) WHERE NOT ignore(roundDown(rand() % 100, [0, 1, 18, 25, 35, 45, 55]))</query>
<query>SELECT count() FROM numbers(10000000) WHERE NOT ignore(roundDown(rand() % 65536, (SELECT groupArray(number) FROM numbers(65536))))</query>
</test>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3">
<test max_ignored_relative_change="0.2">
<settings>
<output_format_pretty_max_rows>1000000</output_format_pretty_max_rows>
<max_threads>1</max_threads>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test>
<substitutions>
<substitution>
<name>table_small</name>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.7">
<test>
<create_query>CREATE TABLE test_in (`a` UInt32) ENGINE = MergeTree() ORDER BY a</create_query>
<fill_query>INSERT INTO test_in SELECT number FROM numbers(500000000)</fill_query>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2">
<test>
<create_query>DROP TABLE IF EXISTS perf_lc_fixed_str_groupby</create_query>
<create_query>CREATE TABLE perf_lc_fixed_str_groupby(
a LowCardinality(FixedString(14)),

View File

@ -1,4 +1,4 @@
<test>
<test max_ignored_relative_change="0.2">

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.4">
<test>
<preconditions>
<table_exists>hits_10m_single</table_exists>
<table_exists>hits_100m_single</table_exists>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3">
<test>
<settings>
<max_threads>1</max_threads>
</settings>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.9">
<test max_ignored_relative_change="0.7">
<preconditions>
<table_exists>hits_100m_single</table_exists>

View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.4">
<test max_ignored_relative_change="0.2">
<preconditions>
<table_exists>hits_10m_single</table_exists>