mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 07:01:59 +00:00
move Check Descriptions to commit_status_helper
This commit is contained in:
Max K
Max Kainov
parent
3dedd8d76b
commit
67bddde628
@ -5,12 +5,6 @@ rules:
|
||||
indentation:
|
||||
level: warning
|
||||
indent-sequences: consistent
|
||||
line-length:
|
||||
# there are:
|
||||
# - bash -c "", so this is OK
|
||||
# - yaml in tests
|
||||
max: 1000
|
||||
level: warning
|
||||
comments:
|
||||
min-spaces-from-content: 1
|
||||
document-start: disable
|
||||
|
||||
@ -20,12 +20,10 @@ class CI:
|
||||
from ci_definitions import BuildConfig as BuildConfig
|
||||
from ci_definitions import DigestConfig as DigestConfig
|
||||
from ci_definitions import JobConfig as JobConfig
|
||||
from ci_definitions import CheckDescription as CheckDescription
|
||||
from ci_definitions import Tags as Tags
|
||||
from ci_definitions import JobNames as JobNames
|
||||
from ci_definitions import BuildNames as BuildNames
|
||||
from ci_definitions import StatusNames as StatusNames
|
||||
from ci_definitions import CHECK_DESCRIPTIONS as CHECK_DESCRIPTIONS
|
||||
from ci_definitions import REQUIRED_CHECKS as REQUIRED_CHECKS
|
||||
from ci_definitions import SyncState as SyncState
|
||||
from ci_definitions import MQ_JOBS as MQ_JOBS
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import copy
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable, List, Union, Iterable, Optional, Literal, Any
|
||||
from typing import List, Union, Iterable, Optional, Literal, Any
|
||||
|
||||
from ci_utils import WithIter
|
||||
from integration_test_images import IMAGES
|
||||
@ -646,188 +646,3 @@ MQ_JOBS = [
|
||||
BuildNames.BINARY_RELEASE,
|
||||
JobNames.UNIT_TEST,
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class CheckDescription:
|
||||
name: str
|
||||
description: str # the check descriptions, will be put into the status table
|
||||
match_func: Callable[[str], bool] # the function to check vs the commit status
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(self.name + self.description)
|
||||
|
||||
|
||||
CHECK_DESCRIPTIONS = [
|
||||
CheckDescription(
|
||||
StatusNames.PR_CHECK,
|
||||
"Checks correctness of the PR's body",
|
||||
lambda x: x == "PR Check",
|
||||
),
|
||||
CheckDescription(
|
||||
StatusNames.SYNC,
|
||||
"If it fails, ask a maintainer for help",
|
||||
lambda x: x == StatusNames.SYNC,
|
||||
),
|
||||
CheckDescription(
|
||||
"AST fuzzer",
|
||||
"Runs randomly generated queries to catch program errors. "
|
||||
"The build type is optionally given in parenthesis. "
|
||||
"If it fails, ask a maintainer for help",
|
||||
lambda x: x.startswith("AST fuzzer"),
|
||||
),
|
||||
CheckDescription(
|
||||
JobNames.BUGFIX_VALIDATE,
|
||||
"Checks that either a new test (functional or integration) or there "
|
||||
"some changed tests that fail with the binary built on master branch",
|
||||
lambda x: x == JobNames.BUGFIX_VALIDATE,
|
||||
),
|
||||
CheckDescription(
|
||||
StatusNames.CI,
|
||||
"A meta-check that indicates the running CI. Normally, it's in <b>success</b> or "
|
||||
"<b>pending</b> state. The failed status indicates some problems with the PR",
|
||||
lambda x: x == "CI running",
|
||||
),
|
||||
CheckDescription(
|
||||
"Builds",
|
||||
"Builds ClickHouse in various configurations for use in further steps. "
|
||||
"You have to fix the builds that fail. Build logs often has enough "
|
||||
"information to fix the error, but you might have to reproduce the failure "
|
||||
"locally. The <b>cmake</b> options can be found in the build log, grepping for "
|
||||
'<b>cmake</b>. Use these options and follow the <a href="'
|
||||
'https://clickhouse.com/docs/en/development/build">general build process</a>',
|
||||
lambda x: x.startswith("ClickHouse") and x.endswith("build check"),
|
||||
),
|
||||
CheckDescription(
|
||||
"Compatibility check",
|
||||
"Checks that <b>clickhouse</b> binary runs on distributions with old libc "
|
||||
"versions. If it fails, ask a maintainer for help",
|
||||
lambda x: x.startswith("Compatibility check"),
|
||||
),
|
||||
CheckDescription(
|
||||
JobNames.DOCKER_SERVER,
|
||||
"The check to build and optionally push the mentioned image to docker hub",
|
||||
lambda x: x.startswith("Docker server"),
|
||||
),
|
||||
CheckDescription(
|
||||
JobNames.DOCKER_KEEPER,
|
||||
"The check to build and optionally push the mentioned image to docker hub",
|
||||
lambda x: x.startswith("Docker keeper"),
|
||||
),
|
||||
CheckDescription(
|
||||
JobNames.DOCS_CHECK,
|
||||
"Builds and tests the documentation",
|
||||
lambda x: x == JobNames.DOCS_CHECK,
|
||||
),
|
||||
CheckDescription(
|
||||
JobNames.FAST_TEST,
|
||||
"Normally this is the first check that is ran for a PR. It builds ClickHouse "
|
||||
'and runs most of <a href="https://clickhouse.com/docs/en/development/tests'
|
||||
'#functional-tests">stateless functional tests</a>, '
|
||||
"omitting some. If it fails, further checks are not started until it is fixed. "
|
||||
"Look at the report to see which tests fail, then reproduce the failure "
|
||||
'locally as described <a href="https://clickhouse.com/docs/en/development/'
|
||||
'tests#functional-test-locally">here</a>',
|
||||
lambda x: x == JobNames.FAST_TEST,
|
||||
),
|
||||
CheckDescription(
|
||||
"Flaky tests",
|
||||
"Checks if new added or modified tests are flaky by running them repeatedly, "
|
||||
"in parallel, with more randomization. Functional tests are run 100 times "
|
||||
"with address sanitizer, and additional randomization of thread scheduling. "
|
||||
"Integration tests are run up to 10 times. If at least once a new test has "
|
||||
"failed, or was too long, this check will be red. We don't allow flaky tests, "
|
||||
'read <a href="https://clickhouse.com/blog/decorating-a-christmas-tree-with-'
|
||||
'the-help-of-flaky-tests/">the doc</a>',
|
||||
lambda x: "tests flaky check" in x,
|
||||
),
|
||||
CheckDescription(
|
||||
"Install packages",
|
||||
"Checks that the built packages are installable in a clear environment",
|
||||
lambda x: x.startswith("Install packages ("),
|
||||
),
|
||||
CheckDescription(
|
||||
"Integration tests",
|
||||
"The integration tests report. In parenthesis the package type is given, "
|
||||
"and in square brackets are the optional part/total tests",
|
||||
lambda x: x.startswith("Integration tests ("),
|
||||
),
|
||||
CheckDescription(
|
||||
StatusNames.MERGEABLE,
|
||||
"Checks if all other necessary checks are successful",
|
||||
lambda x: x == StatusNames.MERGEABLE,
|
||||
),
|
||||
CheckDescription(
|
||||
"Performance Comparison",
|
||||
"Measure changes in query performance. The performance test report is "
|
||||
'described in detail <a href="https://github.com/ClickHouse/ClickHouse/tree'
|
||||
'/master/docker/test/performance-comparison#how-to-read-the-report">here</a>. '
|
||||
"In square brackets are the optional part/total tests",
|
||||
lambda x: x.startswith("Performance Comparison"),
|
||||
),
|
||||
CheckDescription(
|
||||
"Push to Dockerhub",
|
||||
"The check for building and pushing the CI related docker images to docker hub",
|
||||
lambda x: x.startswith("Push") and "to Dockerhub" in x,
|
||||
),
|
||||
CheckDescription(
|
||||
"Sqllogic",
|
||||
"Run clickhouse on the "
|
||||
'<a href="https://www.sqlite.org/sqllogictest">sqllogic</a> '
|
||||
"test set against sqlite and checks that all statements are passed",
|
||||
lambda x: x.startswith("Sqllogic test"),
|
||||
),
|
||||
CheckDescription(
|
||||
"SQLancer",
|
||||
"Fuzzing tests that detect logical bugs with "
|
||||
'<a href="https://github.com/sqlancer/sqlancer">SQLancer</a> tool',
|
||||
lambda x: x.startswith("SQLancer"),
|
||||
),
|
||||
CheckDescription(
|
||||
"Stateful tests",
|
||||
"Runs stateful functional tests for ClickHouse binaries built in various "
|
||||
"configurations -- release, debug, with sanitizers, etc",
|
||||
lambda x: x.startswith("Stateful tests ("),
|
||||
),
|
||||
CheckDescription(
|
||||
"Stateless tests",
|
||||
"Runs stateless functional tests for ClickHouse binaries built in various "
|
||||
"configurations -- release, debug, with sanitizers, etc",
|
||||
lambda x: x.startswith("Stateless tests ("),
|
||||
),
|
||||
CheckDescription(
|
||||
"Stress test",
|
||||
"Runs stateless functional tests concurrently from several clients to detect "
|
||||
"concurrency-related errors",
|
||||
lambda x: x.startswith("Stress test ("),
|
||||
),
|
||||
CheckDescription(
|
||||
JobNames.STYLE_CHECK,
|
||||
"Runs a set of checks to keep the code style clean. If some of tests failed, "
|
||||
"see the related log from the report",
|
||||
lambda x: x == JobNames.STYLE_CHECK,
|
||||
),
|
||||
CheckDescription(
|
||||
"Unit tests",
|
||||
"Runs the unit tests for different release types",
|
||||
lambda x: x.startswith("Unit tests ("),
|
||||
),
|
||||
CheckDescription(
|
||||
"Upgrade check",
|
||||
"Runs stress tests on server version from last release and then tries to "
|
||||
"upgrade it to the version from the PR. It checks if the new server can "
|
||||
"successfully startup without any errors, crashes or sanitizer asserts",
|
||||
lambda x: x.startswith("Upgrade check ("),
|
||||
),
|
||||
CheckDescription(
|
||||
"ClickBench",
|
||||
"Runs [ClickBench](https://github.com/ClickHouse/ClickBench/) with instant-attach table",
|
||||
lambda x: x.startswith("ClickBench"),
|
||||
),
|
||||
CheckDescription(
|
||||
"Fallback for unknown",
|
||||
"There's no description for the check yet, please add it to "
|
||||
"tests/ci/ci_config.py:CHECK_DESCRIPTIONS",
|
||||
lambda x: True,
|
||||
),
|
||||
]
|
||||
|
||||
@ -7,7 +7,7 @@ import time
|
||||
from collections import defaultdict
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Union
|
||||
from typing import Dict, List, Optional, Union, Callable
|
||||
|
||||
from github import Github
|
||||
from github.Commit import Commit
|
||||
@ -176,7 +176,7 @@ def set_status_comment(commit: Commit, pr_info: PRInfo) -> None:
|
||||
|
||||
if not [status for status in statuses if status.context == CI.StatusNames.CI]:
|
||||
# This is the case, when some statuses already exist for the check,
|
||||
# but not the StatusNames.CI. We should create it as pending.
|
||||
# but not the CI.StatusNames.CI. We should create it as pending.
|
||||
# W/o pr_info to avoid recursion, and yes, one extra create_ci_report
|
||||
post_commit_status(
|
||||
commit,
|
||||
@ -226,20 +226,20 @@ def generate_status_comment(pr_info: PRInfo, statuses: CommitStatuses) -> str:
|
||||
f"\n"
|
||||
)
|
||||
# group checks by the name to get the worst one per each
|
||||
grouped_statuses = {} # type: Dict[CI.CheckDescription, CommitStatuses]
|
||||
grouped_statuses = {} # type: Dict[CheckDescription, CommitStatuses]
|
||||
for status in statuses:
|
||||
cd = None
|
||||
for c in CI.CHECK_DESCRIPTIONS:
|
||||
for c in CHECK_DESCRIPTIONS:
|
||||
if c.match_func(status.context):
|
||||
cd = c
|
||||
break
|
||||
|
||||
if cd is None or cd == CI.CHECK_DESCRIPTIONS[-1]:
|
||||
if cd is None or cd == CHECK_DESCRIPTIONS[-1]:
|
||||
# This is the case for either non-found description or a fallback
|
||||
cd = CI.CheckDescription(
|
||||
cd = CheckDescription(
|
||||
status.context,
|
||||
CI.CHECK_DESCRIPTIONS[-1].description,
|
||||
CI.CHECK_DESCRIPTIONS[-1].match_func,
|
||||
CHECK_DESCRIPTIONS[-1].description,
|
||||
CHECK_DESCRIPTIONS[-1].match_func,
|
||||
)
|
||||
|
||||
if cd in grouped_statuses:
|
||||
@ -459,7 +459,7 @@ def trigger_mergeable_check(
|
||||
set_from_sync: bool = False,
|
||||
workflow_failed: bool = False,
|
||||
) -> StatusType:
|
||||
"""calculate and update StatusNames.MERGEABLE"""
|
||||
"""calculate and update CI.StatusNames.MERGEABLE"""
|
||||
required_checks = [status for status in statuses if CI.is_required(status.context)]
|
||||
|
||||
mergeable_status = None
|
||||
@ -536,3 +536,188 @@ def update_upstream_sync_status(
|
||||
get_commit_filtered_statuses(last_synced_upstream_commit),
|
||||
set_from_sync=True,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CheckDescription:
|
||||
name: str
|
||||
description: str # the check descriptions, will be put into the status table
|
||||
match_func: Callable[[str], bool] # the function to check vs the commit status
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(self.name + self.description)
|
||||
|
||||
|
||||
CHECK_DESCRIPTIONS = [
|
||||
CheckDescription(
|
||||
CI.StatusNames.PR_CHECK,
|
||||
"Checks correctness of the PR's body",
|
||||
lambda x: x == "PR Check",
|
||||
),
|
||||
CheckDescription(
|
||||
CI.StatusNames.SYNC,
|
||||
"If it fails, ask a maintainer for help",
|
||||
lambda x: x == CI.StatusNames.SYNC,
|
||||
),
|
||||
CheckDescription(
|
||||
"AST fuzzer",
|
||||
"Runs randomly generated queries to catch program errors. "
|
||||
"The build type is optionally given in parenthesis. "
|
||||
"If it fails, ask a maintainer for help",
|
||||
lambda x: x.startswith("AST fuzzer"),
|
||||
),
|
||||
CheckDescription(
|
||||
CI.JobNames.BUGFIX_VALIDATE,
|
||||
"Checks that either a new test (functional or integration) or there "
|
||||
"some changed tests that fail with the binary built on master branch",
|
||||
lambda x: x == CI.JobNames.BUGFIX_VALIDATE,
|
||||
),
|
||||
CheckDescription(
|
||||
CI.StatusNames.CI,
|
||||
"A meta-check that indicates the running CI. Normally, it's in <b>success</b> or "
|
||||
"<b>pending</b> state. The failed status indicates some problems with the PR",
|
||||
lambda x: x == "CI running",
|
||||
),
|
||||
CheckDescription(
|
||||
"Builds",
|
||||
"Builds ClickHouse in various configurations for use in further steps. "
|
||||
"You have to fix the builds that fail. Build logs often has enough "
|
||||
"information to fix the error, but you might have to reproduce the failure "
|
||||
"locally. The <b>cmake</b> options can be found in the build log, grepping for "
|
||||
'<b>cmake</b>. Use these options and follow the <a href="'
|
||||
'https://clickhouse.com/docs/en/development/build">general build process</a>',
|
||||
lambda x: x.startswith("ClickHouse") and x.endswith("build check"),
|
||||
),
|
||||
CheckDescription(
|
||||
"Compatibility check",
|
||||
"Checks that <b>clickhouse</b> binary runs on distributions with old libc "
|
||||
"versions. If it fails, ask a maintainer for help",
|
||||
lambda x: x.startswith("Compatibility check"),
|
||||
),
|
||||
CheckDescription(
|
||||
CI.JobNames.DOCKER_SERVER,
|
||||
"The check to build and optionally push the mentioned image to docker hub",
|
||||
lambda x: x.startswith("Docker server"),
|
||||
),
|
||||
CheckDescription(
|
||||
CI.JobNames.DOCKER_KEEPER,
|
||||
"The check to build and optionally push the mentioned image to docker hub",
|
||||
lambda x: x.startswith("Docker keeper"),
|
||||
),
|
||||
CheckDescription(
|
||||
CI.JobNames.DOCS_CHECK,
|
||||
"Builds and tests the documentation",
|
||||
lambda x: x == CI.JobNames.DOCS_CHECK,
|
||||
),
|
||||
CheckDescription(
|
||||
CI.JobNames.FAST_TEST,
|
||||
"Normally this is the first check that is ran for a PR. It builds ClickHouse "
|
||||
'and runs most of <a href="https://clickhouse.com/docs/en/development/tests'
|
||||
'#functional-tests">stateless functional tests</a>, '
|
||||
"omitting some. If it fails, further checks are not started until it is fixed. "
|
||||
"Look at the report to see which tests fail, then reproduce the failure "
|
||||
'locally as described <a href="https://clickhouse.com/docs/en/development/'
|
||||
'tests#functional-test-locally">here</a>',
|
||||
lambda x: x == CI.JobNames.FAST_TEST,
|
||||
),
|
||||
CheckDescription(
|
||||
"Flaky tests",
|
||||
"Checks if new added or modified tests are flaky by running them repeatedly, "
|
||||
"in parallel, with more randomization. Functional tests are run 100 times "
|
||||
"with address sanitizer, and additional randomization of thread scheduling. "
|
||||
"Integration tests are run up to 10 times. If at least once a new test has "
|
||||
"failed, or was too long, this check will be red. We don't allow flaky tests, "
|
||||
'read <a href="https://clickhouse.com/blog/decorating-a-christmas-tree-with-'
|
||||
'the-help-of-flaky-tests/">the doc</a>',
|
||||
lambda x: "tests flaky check" in x,
|
||||
),
|
||||
CheckDescription(
|
||||
"Install packages",
|
||||
"Checks that the built packages are installable in a clear environment",
|
||||
lambda x: x.startswith("Install packages ("),
|
||||
),
|
||||
CheckDescription(
|
||||
"Integration tests",
|
||||
"The integration tests report. In parenthesis the package type is given, "
|
||||
"and in square brackets are the optional part/total tests",
|
||||
lambda x: x.startswith("Integration tests ("),
|
||||
),
|
||||
CheckDescription(
|
||||
CI.StatusNames.MERGEABLE,
|
||||
"Checks if all other necessary checks are successful",
|
||||
lambda x: x == CI.StatusNames.MERGEABLE,
|
||||
),
|
||||
CheckDescription(
|
||||
"Performance Comparison",
|
||||
"Measure changes in query performance. The performance test report is "
|
||||
'described in detail <a href="https://github.com/ClickHouse/ClickHouse/tree'
|
||||
'/master/docker/test/performance-comparison#how-to-read-the-report">here</a>. '
|
||||
"In square brackets are the optional part/total tests",
|
||||
lambda x: x.startswith("Performance Comparison"),
|
||||
),
|
||||
CheckDescription(
|
||||
"Push to Dockerhub",
|
||||
"The check for building and pushing the CI related docker images to docker hub",
|
||||
lambda x: x.startswith("Push") and "to Dockerhub" in x,
|
||||
),
|
||||
CheckDescription(
|
||||
"Sqllogic",
|
||||
"Run clickhouse on the "
|
||||
'<a href="https://www.sqlite.org/sqllogictest">sqllogic</a> '
|
||||
"test set against sqlite and checks that all statements are passed",
|
||||
lambda x: x.startswith("Sqllogic test"),
|
||||
),
|
||||
CheckDescription(
|
||||
"SQLancer",
|
||||
"Fuzzing tests that detect logical bugs with "
|
||||
'<a href="https://github.com/sqlancer/sqlancer">SQLancer</a> tool',
|
||||
lambda x: x.startswith("SQLancer"),
|
||||
),
|
||||
CheckDescription(
|
||||
"Stateful tests",
|
||||
"Runs stateful functional tests for ClickHouse binaries built in various "
|
||||
"configurations -- release, debug, with sanitizers, etc",
|
||||
lambda x: x.startswith("Stateful tests ("),
|
||||
),
|
||||
CheckDescription(
|
||||
"Stateless tests",
|
||||
"Runs stateless functional tests for ClickHouse binaries built in various "
|
||||
"configurations -- release, debug, with sanitizers, etc",
|
||||
lambda x: x.startswith("Stateless tests ("),
|
||||
),
|
||||
CheckDescription(
|
||||
"Stress test",
|
||||
"Runs stateless functional tests concurrently from several clients to detect "
|
||||
"concurrency-related errors",
|
||||
lambda x: x.startswith("Stress test ("),
|
||||
),
|
||||
CheckDescription(
|
||||
CI.JobNames.STYLE_CHECK,
|
||||
"Runs a set of checks to keep the code style clean. If some of tests failed, "
|
||||
"see the related log from the report",
|
||||
lambda x: x == CI.JobNames.STYLE_CHECK,
|
||||
),
|
||||
CheckDescription(
|
||||
"Unit tests",
|
||||
"Runs the unit tests for different release types",
|
||||
lambda x: x.startswith("Unit tests ("),
|
||||
),
|
||||
CheckDescription(
|
||||
"Upgrade check",
|
||||
"Runs stress tests on server version from last release and then tries to "
|
||||
"upgrade it to the version from the PR. It checks if the new server can "
|
||||
"successfully startup without any errors, crashes or sanitizer asserts",
|
||||
lambda x: x.startswith("Upgrade check ("),
|
||||
),
|
||||
CheckDescription(
|
||||
"ClickBench",
|
||||
"Runs [ClickBench](https://github.com/ClickHouse/ClickBench/) with instant-attach table",
|
||||
lambda x: x.startswith("ClickBench"),
|
||||
),
|
||||
CheckDescription(
|
||||
"Fallback for unknown",
|
||||
"There's no description for the check yet, please add it to "
|
||||
"tests/ci/ci_config.py:CHECK_DESCRIPTIONS",
|
||||
lambda x: True,
|
||||
),
|
||||
]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user