thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 08:32:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

move Check Descriptions to commit_status_helper

Browse Source
This commit is contained in:
Max K 2024-08-03 21:26:05 +02:00 committed by Max Kainov
parent 3dedd8d76b
commit 67bddde628
4 changed files with 195 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.yamllint
View File

@ -5,12 +5,6 @@ rules:
indentation: indentation:
level: warning level: warning
indent-sequences: consistent indent-sequences: consistent
line-length:
# there are:
# - bash -c "", so this is OK
# - yaml in tests
max: 1000
level: warning
comments: comments:
min-spaces-from-content: 1 min-spaces-from-content: 1
document-start: disable document-start: disable

2
tests/ci/ci_config.py
View File

@ -20,12 +20,10 @@ class CI:
from ci_definitions import BuildConfig as BuildConfig from ci_definitions import BuildConfig as BuildConfig
from ci_definitions import DigestConfig as DigestConfig from ci_definitions import DigestConfig as DigestConfig
from ci_definitions import JobConfig as JobConfig from ci_definitions import JobConfig as JobConfig
from ci_definitions import CheckDescription as CheckDescription
from ci_definitions import Tags as Tags from ci_definitions import Tags as Tags
from ci_definitions import JobNames as JobNames from ci_definitions import JobNames as JobNames
from ci_definitions import BuildNames as BuildNames from ci_definitions import BuildNames as BuildNames
from ci_definitions import StatusNames as StatusNames from ci_definitions import StatusNames as StatusNames
from ci_definitions import CHECK_DESCRIPTIONS as CHECK_DESCRIPTIONS
from ci_definitions import REQUIRED_CHECKS as REQUIRED_CHECKS from ci_definitions import REQUIRED_CHECKS as REQUIRED_CHECKS
from ci_definitions import SyncState as SyncState from ci_definitions import SyncState as SyncState
from ci_definitions import MQ_JOBS as MQ_JOBS from ci_definitions import MQ_JOBS as MQ_JOBS

187
tests/ci/ci_definitions.py
View File

@ -1,7 +1,7 @@
import copy import copy
from dataclasses import dataclass, field from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from typing import Callable, List, Union, Iterable, Optional, Literal, Any from typing import List, Union, Iterable, Optional, Literal, Any
from ci_utils import WithIter from ci_utils import WithIter
from integration_test_images import IMAGES from integration_test_images import IMAGES
@ -646,188 +646,3 @@ MQ_JOBS = [
BuildNames.BINARY_RELEASE, BuildNames.BINARY_RELEASE,
JobNames.UNIT_TEST, JobNames.UNIT_TEST,
] ]
@dataclass
class CheckDescription:
name: str
description: str # the check descriptions, will be put into the status table
match_func: Callable[[str], bool] # the function to check vs the commit status
def __hash__(self) -> int:
return hash(self.name + self.description)
CHECK_DESCRIPTIONS = [
CheckDescription(
StatusNames.PR_CHECK,
"Checks correctness of the PR's body",
lambda x: x == "PR Check",
),
CheckDescription(
StatusNames.SYNC,
"If it fails, ask a maintainer for help",
lambda x: x == StatusNames.SYNC,
),
CheckDescription(
"AST fuzzer",
"Runs randomly generated queries to catch program errors. "
"The build type is optionally given in parenthesis. "
"If it fails, ask a maintainer for help",
lambda x: x.startswith("AST fuzzer"),
),
CheckDescription(
JobNames.BUGFIX_VALIDATE,
"Checks that either a new test (functional or integration) or there "
"some changed tests that fail with the binary built on master branch",
lambda x: x == JobNames.BUGFIX_VALIDATE,
),
CheckDescription(
StatusNames.CI,
"A meta-check that indicates the running CI. Normally, it's in <b>success</b> or "
"<b>pending</b> state. The failed status indicates some problems with the PR",
lambda x: x == "CI running",
),
CheckDescription(
"Builds",
"Builds ClickHouse in various configurations for use in further steps. "
"You have to fix the builds that fail. Build logs often has enough "
"information to fix the error, but you might have to reproduce the failure "
"locally. The <b>cmake</b> options can be found in the build log, grepping for "
'<b>cmake</b>. Use these options and follow the <a href="'
'https://clickhouse.com/docs/en/development/build">general build process</a>',
lambda x: x.startswith("ClickHouse") and x.endswith("build check"),
),
CheckDescription(
"Compatibility check",
"Checks that <b>clickhouse</b> binary runs on distributions with old libc "
"versions. If it fails, ask a maintainer for help",
lambda x: x.startswith("Compatibility check"),
),
CheckDescription(
JobNames.DOCKER_SERVER,
"The check to build and optionally push the mentioned image to docker hub",
lambda x: x.startswith("Docker server"),
),
CheckDescription(
JobNames.DOCKER_KEEPER,
"The check to build and optionally push the mentioned image to docker hub",
lambda x: x.startswith("Docker keeper"),
),
CheckDescription(
JobNames.DOCS_CHECK,
"Builds and tests the documentation",
lambda x: x == JobNames.DOCS_CHECK,
),
CheckDescription(
JobNames.FAST_TEST,
"Normally this is the first check that is ran for a PR. It builds ClickHouse "
'and runs most of <a href="https://clickhouse.com/docs/en/development/tests'
'#functional-tests">stateless functional tests</a>, '
"omitting some. If it fails, further checks are not started until it is fixed. "
"Look at the report to see which tests fail, then reproduce the failure "
'locally as described <a href="https://clickhouse.com/docs/en/development/'
'tests#functional-test-locally">here</a>',
lambda x: x == JobNames.FAST_TEST,
),
CheckDescription(
"Flaky tests",
"Checks if new added or modified tests are flaky by running them repeatedly, "
"in parallel, with more randomization. Functional tests are run 100 times "
"with address sanitizer, and additional randomization of thread scheduling. "
"Integration tests are run up to 10 times. If at least once a new test has "
"failed, or was too long, this check will be red. We don't allow flaky tests, "
'read <a href="https://clickhouse.com/blog/decorating-a-christmas-tree-with-'
'the-help-of-flaky-tests/">the doc</a>',
lambda x: "tests flaky check" in x,
),
CheckDescription(
"Install packages",
"Checks that the built packages are installable in a clear environment",
lambda x: x.startswith("Install packages ("),
),
CheckDescription(
"Integration tests",
"The integration tests report. In parenthesis the package type is given, "
"and in square brackets are the optional part/total tests",
lambda x: x.startswith("Integration tests ("),
),
CheckDescription(
StatusNames.MERGEABLE,
"Checks if all other necessary checks are successful",
lambda x: x == StatusNames.MERGEABLE,
),
CheckDescription(
"Performance Comparison",
"Measure changes in query performance. The performance test report is "
'described in detail <a href="https://github.com/ClickHouse/ClickHouse/tree'
'/master/docker/test/performance-comparison#how-to-read-the-report">here</a>. '
"In square brackets are the optional part/total tests",
lambda x: x.startswith("Performance Comparison"),
),
CheckDescription(
"Push to Dockerhub",
"The check for building and pushing the CI related docker images to docker hub",
lambda x: x.startswith("Push") and "to Dockerhub" in x,
),
CheckDescription(
"Sqllogic",
"Run clickhouse on the "
'<a href="https://www.sqlite.org/sqllogictest">sqllogic</a> '
"test set against sqlite and checks that all statements are passed",
lambda x: x.startswith("Sqllogic test"),
),
CheckDescription(
"SQLancer",
"Fuzzing tests that detect logical bugs with "
'<a href="https://github.com/sqlancer/sqlancer">SQLancer</a> tool',
lambda x: x.startswith("SQLancer"),
),
CheckDescription(
"Stateful tests",
"Runs stateful functional tests for ClickHouse binaries built in various "
"configurations -- release, debug, with sanitizers, etc",
lambda x: x.startswith("Stateful tests ("),
),
CheckDescription(
"Stateless tests",
"Runs stateless functional tests for ClickHouse binaries built in various "
"configurations -- release, debug, with sanitizers, etc",
lambda x: x.startswith("Stateless tests ("),
),
CheckDescription(
"Stress test",
"Runs stateless functional tests concurrently from several clients to detect "
"concurrency-related errors",
lambda x: x.startswith("Stress test ("),
),
CheckDescription(
JobNames.STYLE_CHECK,
"Runs a set of checks to keep the code style clean. If some of tests failed, "
"see the related log from the report",
lambda x: x == JobNames.STYLE_CHECK,
),
CheckDescription(
"Unit tests",
"Runs the unit tests for different release types",
lambda x: x.startswith("Unit tests ("),
),
CheckDescription(
"Upgrade check",
"Runs stress tests on server version from last release and then tries to "
"upgrade it to the version from the PR. It checks if the new server can "
"successfully startup without any errors, crashes or sanitizer asserts",
lambda x: x.startswith("Upgrade check ("),
),
CheckDescription(
"ClickBench",
"Runs [ClickBench](https://github.com/ClickHouse/ClickBench/) with instant-attach table",
lambda x: x.startswith("ClickBench"),
),
CheckDescription(
"Fallback for unknown",
"There's no description for the check yet, please add it to "
"tests/ci/ci_config.py:CHECK_DESCRIPTIONS",
lambda x: True,
),
]

203
tests/ci/commit_status_helper.py
View File

@ -7,7 +7,7 @@ import time
from collections import defaultdict from collections import defaultdict
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
from pathlib import Path from pathlib import Path
from typing import Dict, List, Optional, Union from typing import Dict, List, Optional, Union, Callable
from github import Github from github import Github
from github.Commit import Commit from github.Commit import Commit
@ -176,7 +176,7 @@ def set_status_comment(commit: Commit, pr_info: PRInfo) -> None:
if not [status for status in statuses if status.context == CI.StatusNames.CI]: if not [status for status in statuses if status.context == CI.StatusNames.CI]:
# This is the case, when some statuses already exist for the check, # This is the case, when some statuses already exist for the check,
# but not the StatusNames.CI. We should create it as pending. # but not the CI.StatusNames.CI. We should create it as pending.
# W/o pr_info to avoid recursion, and yes, one extra create_ci_report # W/o pr_info to avoid recursion, and yes, one extra create_ci_report
post_commit_status( post_commit_status(
commit, commit,
@ -226,20 +226,20 @@ def generate_status_comment(pr_info: PRInfo, statuses: CommitStatuses) -> str:
f"\n" f"\n"
) )
# group checks by the name to get the worst one per each # group checks by the name to get the worst one per each
grouped_statuses = {} # type: Dict[CI.CheckDescription, CommitStatuses] grouped_statuses = {} # type: Dict[CheckDescription, CommitStatuses]
for status in statuses: for status in statuses:
cd = None cd = None
for c in CI.CHECK_DESCRIPTIONS: for c in CHECK_DESCRIPTIONS:
if c.match_func(status.context): if c.match_func(status.context):
cd = c cd = c
break break
if cd is None or cd == CI.CHECK_DESCRIPTIONS[-1]: if cd is None or cd == CHECK_DESCRIPTIONS[-1]:
# This is the case for either non-found description or a fallback # This is the case for either non-found description or a fallback
cd = CI.CheckDescription( cd = CheckDescription(
status.context, status.context,
CI.CHECK_DESCRIPTIONS[-1].description, CHECK_DESCRIPTIONS[-1].description,
CI.CHECK_DESCRIPTIONS[-1].match_func, CHECK_DESCRIPTIONS[-1].match_func,
) )
if cd in grouped_statuses: if cd in grouped_statuses:
@ -459,7 +459,7 @@ def trigger_mergeable_check(
set_from_sync: bool = False, set_from_sync: bool = False,
workflow_failed: bool = False, workflow_failed: bool = False,
) -> StatusType: ) -> StatusType:
"""calculate and update StatusNames.MERGEABLE""" """calculate and update CI.StatusNames.MERGEABLE"""
required_checks = [status for status in statuses if CI.is_required(status.context)] required_checks = [status for status in statuses if CI.is_required(status.context)]
mergeable_status = None mergeable_status = None
@ -536,3 +536,188 @@ def update_upstream_sync_status(
get_commit_filtered_statuses(last_synced_upstream_commit), get_commit_filtered_statuses(last_synced_upstream_commit),
set_from_sync=True, set_from_sync=True,
) )
@dataclass
class CheckDescription:
name: str
description: str # the check descriptions, will be put into the status table
match_func: Callable[[str], bool] # the function to check vs the commit status
def __hash__(self) -> int:
return hash(self.name + self.description)
CHECK_DESCRIPTIONS = [
CheckDescription(
CI.StatusNames.PR_CHECK,
"Checks correctness of the PR's body",
lambda x: x == "PR Check",
),
CheckDescription(
CI.StatusNames.SYNC,
"If it fails, ask a maintainer for help",
lambda x: x == CI.StatusNames.SYNC,
),
CheckDescription(
"AST fuzzer",
"Runs randomly generated queries to catch program errors. "
"The build type is optionally given in parenthesis. "
"If it fails, ask a maintainer for help",
lambda x: x.startswith("AST fuzzer"),
),
CheckDescription(
CI.JobNames.BUGFIX_VALIDATE,
"Checks that either a new test (functional or integration) or there "
"some changed tests that fail with the binary built on master branch",
lambda x: x == CI.JobNames.BUGFIX_VALIDATE,
),
CheckDescription(
CI.StatusNames.CI,
"A meta-check that indicates the running CI. Normally, it's in <b>success</b> or "
"<b>pending</b> state. The failed status indicates some problems with the PR",
lambda x: x == "CI running",
),
CheckDescription(
"Builds",
"Builds ClickHouse in various configurations for use in further steps. "
"You have to fix the builds that fail. Build logs often has enough "
"information to fix the error, but you might have to reproduce the failure "
"locally. The <b>cmake</b> options can be found in the build log, grepping for "
'<b>cmake</b>. Use these options and follow the <a href="'
'https://clickhouse.com/docs/en/development/build">general build process</a>',
lambda x: x.startswith("ClickHouse") and x.endswith("build check"),
),
CheckDescription(
"Compatibility check",
"Checks that <b>clickhouse</b> binary runs on distributions with old libc "
"versions. If it fails, ask a maintainer for help",
lambda x: x.startswith("Compatibility check"),
),
CheckDescription(
CI.JobNames.DOCKER_SERVER,
"The check to build and optionally push the mentioned image to docker hub",
lambda x: x.startswith("Docker server"),
),
CheckDescription(
CI.JobNames.DOCKER_KEEPER,
"The check to build and optionally push the mentioned image to docker hub",
lambda x: x.startswith("Docker keeper"),
),
CheckDescription(
CI.JobNames.DOCS_CHECK,
"Builds and tests the documentation",
lambda x: x == CI.JobNames.DOCS_CHECK,
),
CheckDescription(
CI.JobNames.FAST_TEST,
"Normally this is the first check that is ran for a PR. It builds ClickHouse "
'and runs most of <a href="https://clickhouse.com/docs/en/development/tests'
'#functional-tests">stateless functional tests</a>, '
"omitting some. If it fails, further checks are not started until it is fixed. "
"Look at the report to see which tests fail, then reproduce the failure "
'locally as described <a href="https://clickhouse.com/docs/en/development/'
'tests#functional-test-locally">here</a>',
lambda x: x == CI.JobNames.FAST_TEST,
),
CheckDescription(
"Flaky tests",
"Checks if new added or modified tests are flaky by running them repeatedly, "
"in parallel, with more randomization. Functional tests are run 100 times "
"with address sanitizer, and additional randomization of thread scheduling. "
"Integration tests are run up to 10 times. If at least once a new test has "
"failed, or was too long, this check will be red. We don't allow flaky tests, "
'read <a href="https://clickhouse.com/blog/decorating-a-christmas-tree-with-'
'the-help-of-flaky-tests/">the doc</a>',
lambda x: "tests flaky check" in x,
),
CheckDescription(
"Install packages",
"Checks that the built packages are installable in a clear environment",
lambda x: x.startswith("Install packages ("),
),
CheckDescription(
"Integration tests",
"The integration tests report. In parenthesis the package type is given, "
"and in square brackets are the optional part/total tests",
lambda x: x.startswith("Integration tests ("),
),
CheckDescription(
CI.StatusNames.MERGEABLE,
"Checks if all other necessary checks are successful",
lambda x: x == CI.StatusNames.MERGEABLE,
),
CheckDescription(
"Performance Comparison",
"Measure changes in query performance. The performance test report is "
'described in detail <a href="https://github.com/ClickHouse/ClickHouse/tree'
'/master/docker/test/performance-comparison#how-to-read-the-report">here</a>. '
"In square brackets are the optional part/total tests",
lambda x: x.startswith("Performance Comparison"),
),
CheckDescription(
"Push to Dockerhub",
"The check for building and pushing the CI related docker images to docker hub",
lambda x: x.startswith("Push") and "to Dockerhub" in x,
),
CheckDescription(
"Sqllogic",
"Run clickhouse on the "
'<a href="https://www.sqlite.org/sqllogictest">sqllogic</a> '
"test set against sqlite and checks that all statements are passed",
lambda x: x.startswith("Sqllogic test"),
),
CheckDescription(
"SQLancer",
"Fuzzing tests that detect logical bugs with "
'<a href="https://github.com/sqlancer/sqlancer">SQLancer</a> tool',
lambda x: x.startswith("SQLancer"),
),
CheckDescription(
"Stateful tests",
"Runs stateful functional tests for ClickHouse binaries built in various "
"configurations -- release, debug, with sanitizers, etc",
lambda x: x.startswith("Stateful tests ("),
),
CheckDescription(
"Stateless tests",
"Runs stateless functional tests for ClickHouse binaries built in various "
"configurations -- release, debug, with sanitizers, etc",
lambda x: x.startswith("Stateless tests ("),
),
CheckDescription(
"Stress test",
"Runs stateless functional tests concurrently from several clients to detect "
"concurrency-related errors",
lambda x: x.startswith("Stress test ("),
),
CheckDescription(
CI.JobNames.STYLE_CHECK,
"Runs a set of checks to keep the code style clean. If some of tests failed, "
"see the related log from the report",
lambda x: x == CI.JobNames.STYLE_CHECK,
),
CheckDescription(
"Unit tests",
"Runs the unit tests for different release types",
lambda x: x.startswith("Unit tests ("),
),
CheckDescription(
"Upgrade check",
"Runs stress tests on server version from last release and then tries to "
"upgrade it to the version from the PR. It checks if the new server can "
"successfully startup without any errors, crashes or sanitizer asserts",
lambda x: x.startswith("Upgrade check ("),
),
CheckDescription(
"ClickBench",
"Runs [ClickBench](https://github.com/ClickHouse/ClickBench/) with instant-attach table",
lambda x: x.startswith("ClickBench"),
),
CheckDescription(
"Fallback for unknown",
"There's no description for the check yet, please add it to "
"tests/ci/ci_config.py:CHECK_DESCRIPTIONS",
lambda x: True,
),
]