From 4506299d73a3fbf8fc9446b3eed05fe4d5553c23 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Tue, 30 May 2023 20:53:45 +0200 Subject: [PATCH 1/2] impl --- docker/test/performance-comparison/report.py | 4 +++- tests/ci/performance_comparison_check.py | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 214f2d550b4..a1f2eb9d9ec 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -626,7 +626,9 @@ if args.report == "main": message_array.append(str(faster_queries) + " faster") if slower_queries: - if slower_queries > 3: + # This threshold should be synchronized with the value in https://github.com/ClickHouse/ClickHouse/blob/master/tests/ci/performance_comparison_check.py#L225 + # False positives rate should be < 1%: https://shorturl.at/CDEK8 + if slower_queries > 5: status = "failure" message_array.append(str(slower_queries) + " slower") diff --git a/tests/ci/performance_comparison_check.py b/tests/ci/performance_comparison_check.py index bf5704f31bd..1baf547816f 100644 --- a/tests/ci/performance_comparison_check.py +++ b/tests/ci/performance_comparison_check.py @@ -219,6 +219,12 @@ if __name__ == "__main__": except Exception: traceback.print_exc() + def too_many_slow(msg): + match = re.search("(|.* )(\d+) slower.*", msg) + # This threshold should be synchronized with the value in https://github.com/ClickHouse/ClickHouse/blob/master/docker/test/performance-comparison/report.py#L629 + threshold = 5 + return int(match.group(2).strip()) > threshold if match else False + # Try to fetch status from the report. status = "" message = "" @@ -236,7 +242,7 @@ if __name__ == "__main__": # TODO: Remove me, always green mode for the first time, unless errors status = "success" - if "errors" in message.lower(): + if "errors" in message.lower() or too_many_slow(message.lower()): status = "failure" # TODO: Remove until here except Exception: From e548dce123debf4864348d606629f90844b5e5f8 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Sat, 3 Jun 2023 00:08:47 +0200 Subject: [PATCH 2/2] fix --- tests/ci/performance_comparison_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/performance_comparison_check.py b/tests/ci/performance_comparison_check.py index 1baf547816f..41ace95c350 100644 --- a/tests/ci/performance_comparison_check.py +++ b/tests/ci/performance_comparison_check.py @@ -220,7 +220,7 @@ if __name__ == "__main__": traceback.print_exc() def too_many_slow(msg): - match = re.search("(|.* )(\d+) slower.*", msg) + match = re.search(r"(|.* )(\d+) slower.*", msg) # This threshold should be synchronized with the value in https://github.com/ClickHouse/ClickHouse/blob/master/docker/test/performance-comparison/report.py#L629 threshold = 5 return int(match.group(2).strip()) > threshold if match else False