thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #50361 from nickitat/fail_perf_on_too_many_slow

Browse Source 
Fail perf tests when too many queries slowed down
This commit is contained in:
Sema Checherinda 2023-06-06 11:33:04 +02:00 committed by GitHub
commit 990652119e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docker/test/performance-comparison/report.py
View File

@ -626,7 +626,9 @@ if args.report == "main":
message_array.append(str(faster_queries) + " faster")
if slower_queries:
if slower_queries > 3:
# This threshold should be synchronized with the value in https://github.com/ClickHouse/ClickHouse/blob/master/tests/ci/performance_comparison_check.py#L225
# False positives rate should be < 1%: https://shorturl.at/CDEK8
if slower_queries > 5:
status = "failure"
message_array.append(str(slower_queries) + " slower")

8
tests/ci/performance_comparison_check.py
View File

@ -219,6 +219,12 @@ if __name__ == "__main__":
except Exception:
traceback.print_exc()
def too_many_slow(msg):
match = re.search(r"(|.* )(\d+) slower.*", msg)
# This threshold should be synchronized with the value in https://github.com/ClickHouse/ClickHouse/blob/master/docker/test/performance-comparison/report.py#L629
threshold = 5
return int(match.group(2).strip()) > threshold if match else False
# Try to fetch status from the report.
status = ""
message = ""
@ -236,7 +242,7 @@ if __name__ == "__main__":
# TODO: Remove me, always green mode for the first time, unless errors
status = "success"
if "errors" in message.lower():
if "errors" in message.lower() or too_many_slow(message.lower()):
status = "failure"
# TODO: Remove until here
except Exception: