mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-15 03:53:41 +00:00
567 lines
19 KiB
Python
567 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Callable, Dict, TypeVar
|
|
|
|
ConfValue = TypeVar("ConfValue", str, bool)
|
|
BuildConfig = Dict[str, ConfValue]
|
|
|
|
CI_CONFIG = {
|
|
"build_config": {
|
|
"package_release": {
|
|
"compiler": "clang-16",
|
|
"build_type": "",
|
|
"sanitizer": "",
|
|
"package_type": "deb",
|
|
"static_binary_name": "amd64",
|
|
"additional_pkgs": True,
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
"coverity": {
|
|
"compiler": "clang-16",
|
|
"build_type": "",
|
|
"sanitizer": "",
|
|
"package_type": "coverity",
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
"official": False,
|
|
},
|
|
"package_aarch64": {
|
|
"compiler": "clang-16-aarch64",
|
|
"build_type": "",
|
|
"sanitizer": "",
|
|
"package_type": "deb",
|
|
"static_binary_name": "aarch64",
|
|
"additional_pkgs": True,
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
"package_asan": {
|
|
"compiler": "clang-16",
|
|
"build_type": "",
|
|
"sanitizer": "address",
|
|
"package_type": "deb",
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
"package_ubsan": {
|
|
"compiler": "clang-16",
|
|
"build_type": "",
|
|
"sanitizer": "undefined",
|
|
"package_type": "deb",
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
"package_tsan": {
|
|
"compiler": "clang-16",
|
|
"build_type": "",
|
|
"sanitizer": "thread",
|
|
"package_type": "deb",
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
"package_msan": {
|
|
"compiler": "clang-16",
|
|
"build_type": "",
|
|
"sanitizer": "memory",
|
|
"package_type": "deb",
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
"package_debug": {
|
|
"compiler": "clang-16",
|
|
"build_type": "debug",
|
|
"sanitizer": "",
|
|
"package_type": "deb",
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
"binary_release": {
|
|
"compiler": "clang-16",
|
|
"build_type": "",
|
|
"sanitizer": "",
|
|
"package_type": "binary",
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
"binary_tidy": {
|
|
"compiler": "clang-16",
|
|
"build_type": "debug",
|
|
"sanitizer": "",
|
|
"package_type": "binary",
|
|
"static_binary_name": "debug-amd64",
|
|
"tidy": "enable",
|
|
"with_coverage": False,
|
|
},
|
|
"binary_darwin": {
|
|
"compiler": "clang-16-darwin",
|
|
"build_type": "",
|
|
"sanitizer": "",
|
|
"package_type": "binary",
|
|
"static_binary_name": "macos",
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
"binary_aarch64": {
|
|
"compiler": "clang-16-aarch64",
|
|
"build_type": "",
|
|
"sanitizer": "",
|
|
"package_type": "binary",
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
"binary_aarch64_v80compat": {
|
|
"compiler": "clang-16-aarch64-v80compat",
|
|
"build_type": "",
|
|
"sanitizer": "",
|
|
"package_type": "binary",
|
|
"static_binary_name": "aarch64v80compat",
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
"binary_freebsd": {
|
|
"compiler": "clang-16-freebsd",
|
|
"build_type": "",
|
|
"sanitizer": "",
|
|
"package_type": "binary",
|
|
"static_binary_name": "freebsd",
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
"binary_darwin_aarch64": {
|
|
"compiler": "clang-16-darwin-aarch64",
|
|
"build_type": "",
|
|
"sanitizer": "",
|
|
"package_type": "binary",
|
|
"static_binary_name": "macos-aarch64",
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
"binary_ppc64le": {
|
|
"compiler": "clang-16-ppc64le",
|
|
"build_type": "",
|
|
"sanitizer": "",
|
|
"package_type": "binary",
|
|
"static_binary_name": "powerpc64le",
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
"binary_amd64_compat": {
|
|
"compiler": "clang-16-amd64-compat",
|
|
"build_type": "",
|
|
"sanitizer": "",
|
|
"package_type": "binary",
|
|
"static_binary_name": "amd64compat",
|
|
"tidy": "disable",
|
|
"with_coverage": False,
|
|
},
|
|
},
|
|
"builds_report_config": {
|
|
"ClickHouse build check": [
|
|
"package_release",
|
|
"coverity",
|
|
"package_aarch64",
|
|
"package_asan",
|
|
"package_ubsan",
|
|
"package_tsan",
|
|
"package_msan",
|
|
"package_debug",
|
|
"binary_release",
|
|
],
|
|
"ClickHouse special build check": [
|
|
"binary_tidy",
|
|
"binary_darwin",
|
|
"binary_aarch64",
|
|
"binary_aarch64_v80compat",
|
|
"binary_freebsd",
|
|
"binary_darwin_aarch64",
|
|
"binary_ppc64le",
|
|
"binary_amd64_compat",
|
|
],
|
|
},
|
|
"tests_config": {
|
|
# required_build - build name for artifacts
|
|
# force_tests - force success status for tests
|
|
"Install packages (amd64)": {
|
|
"required_build": "package_release",
|
|
},
|
|
"Install packages (arm64)": {
|
|
"required_build": "package_aarch64",
|
|
},
|
|
"Stateful tests (asan)": {
|
|
"required_build": "package_asan",
|
|
},
|
|
"Stateful tests (tsan)": {
|
|
"required_build": "package_tsan",
|
|
},
|
|
"Stateful tests (msan)": {
|
|
"required_build": "package_msan",
|
|
},
|
|
"Stateful tests (ubsan)": {
|
|
"required_build": "package_ubsan",
|
|
},
|
|
"Stateful tests (debug)": {
|
|
"required_build": "package_debug",
|
|
},
|
|
"Stateful tests (release)": {
|
|
"required_build": "package_release",
|
|
},
|
|
"Stateful tests (aarch64)": {
|
|
"required_build": "package_aarch64",
|
|
},
|
|
"Stateful tests (release, DatabaseOrdinary)": {
|
|
"required_build": "package_release",
|
|
},
|
|
"Stateful tests (release, DatabaseReplicated)": {
|
|
"required_build": "package_release",
|
|
},
|
|
# Stateful tests for parallel replicas
|
|
"Stateful tests (release, ParallelReplicas)": {
|
|
"required_build": "package_release",
|
|
},
|
|
"Stateful tests (debug, ParallelReplicas)": {
|
|
"required_build": "package_debug",
|
|
},
|
|
"Stateful tests (asan, ParallelReplicas)": {
|
|
"required_build": "package_asan",
|
|
},
|
|
"Stateful tests (msan, ParallelReplicas)": {
|
|
"required_build": "package_msan",
|
|
},
|
|
"Stateful tests (ubsan, ParallelReplicas)": {
|
|
"required_build": "package_ubsan",
|
|
},
|
|
"Stateful tests (tsan, ParallelReplicas)": {
|
|
"required_build": "package_tsan",
|
|
},
|
|
# End stateful tests for parallel replicas
|
|
"Stateless tests (asan)": {
|
|
"required_build": "package_asan",
|
|
},
|
|
"Stateless tests (tsan)": {
|
|
"required_build": "package_tsan",
|
|
},
|
|
"Stateless tests (msan)": {
|
|
"required_build": "package_msan",
|
|
},
|
|
"Stateless tests (ubsan)": {
|
|
"required_build": "package_ubsan",
|
|
},
|
|
"Stateless tests (debug)": {
|
|
"required_build": "package_debug",
|
|
},
|
|
"Stateless tests (release)": {
|
|
"required_build": "package_release",
|
|
},
|
|
"Stateless tests (aarch64)": {
|
|
"required_build": "package_aarch64",
|
|
},
|
|
"Stateless tests (release, wide parts enabled)": {
|
|
"required_build": "package_release",
|
|
},
|
|
"Stateless tests (release, analyzer)": {
|
|
"required_build": "package_release",
|
|
},
|
|
"Stateless tests (release, DatabaseOrdinary)": {
|
|
"required_build": "package_release",
|
|
},
|
|
"Stateless tests (release, DatabaseReplicated)": {
|
|
"required_build": "package_release",
|
|
},
|
|
"Stateless tests (release, s3 storage)": {
|
|
"required_build": "package_release",
|
|
},
|
|
"Stateless tests (debug, s3 storage)": {
|
|
"required_build": "package_debug",
|
|
},
|
|
"Stateless tests (tsan, s3 storage)": {
|
|
"required_build": "package_tsan",
|
|
},
|
|
"Stress test (asan)": {
|
|
"required_build": "package_asan",
|
|
},
|
|
"Stress test (tsan)": {
|
|
"required_build": "package_tsan",
|
|
},
|
|
"Stress test (ubsan)": {
|
|
"required_build": "package_ubsan",
|
|
},
|
|
"Stress test (msan)": {
|
|
"required_build": "package_msan",
|
|
},
|
|
"Stress test (debug)": {
|
|
"required_build": "package_debug",
|
|
},
|
|
"Upgrade check (asan)": {
|
|
"required_build": "package_asan",
|
|
},
|
|
"Upgrade check (tsan)": {
|
|
"required_build": "package_tsan",
|
|
},
|
|
"Upgrade check (msan)": {
|
|
"required_build": "package_msan",
|
|
},
|
|
"Upgrade check (debug)": {
|
|
"required_build": "package_debug",
|
|
},
|
|
"Integration tests (asan)": {
|
|
"required_build": "package_asan",
|
|
},
|
|
"Integration tests (tsan)": {
|
|
"required_build": "package_tsan",
|
|
},
|
|
"Integration tests (release)": {
|
|
"required_build": "package_release",
|
|
},
|
|
"Integration tests (msan)": {
|
|
"required_build": "package_msan",
|
|
},
|
|
"Integration tests flaky check (asan)": {
|
|
"required_build": "package_asan",
|
|
},
|
|
"Compatibility check (amd64)": {
|
|
"required_build": "package_release",
|
|
},
|
|
"Compatibility check (aarch64)": {
|
|
"required_build": "package_aarch64",
|
|
},
|
|
"Unit tests (release-clang)": {
|
|
"required_build": "binary_release",
|
|
},
|
|
"Unit tests (asan)": {
|
|
"required_build": "package_asan",
|
|
},
|
|
"Unit tests (msan)": {
|
|
"required_build": "package_msan",
|
|
},
|
|
"Unit tests (tsan)": {
|
|
"required_build": "package_tsan",
|
|
},
|
|
"Unit tests (ubsan)": {
|
|
"required_build": "package_ubsan",
|
|
},
|
|
"AST fuzzer (debug)": {
|
|
"required_build": "package_debug",
|
|
},
|
|
"AST fuzzer (asan)": {
|
|
"required_build": "package_asan",
|
|
},
|
|
"AST fuzzer (msan)": {
|
|
"required_build": "package_msan",
|
|
},
|
|
"AST fuzzer (tsan)": {
|
|
"required_build": "package_tsan",
|
|
},
|
|
"AST fuzzer (ubsan)": {
|
|
"required_build": "package_ubsan",
|
|
},
|
|
"Stateless tests flaky check (asan)": {
|
|
"required_build": "package_asan",
|
|
},
|
|
"ClickHouse Keeper Jepsen": {
|
|
"required_build": "binary_release",
|
|
},
|
|
"ClickHouse Server Jepsen": {
|
|
"required_build": "binary_release",
|
|
},
|
|
"Performance Comparison": {
|
|
"required_build": "package_release",
|
|
"test_grep_exclude_filter": "",
|
|
},
|
|
"Performance Comparison Aarch64": {
|
|
"required_build": "package_aarch64",
|
|
"test_grep_exclude_filter": "",
|
|
},
|
|
"SQLancer (release)": {
|
|
"required_build": "package_release",
|
|
},
|
|
"SQLancer (debug)": {
|
|
"required_build": "package_debug",
|
|
},
|
|
"Sqllogic test (release)": {
|
|
"required_build": "package_release",
|
|
},
|
|
},
|
|
} # type: dict
|
|
|
|
# checks required by Mergeable Check
|
|
REQUIRED_CHECKS = [
|
|
"ClickHouse build check",
|
|
"ClickHouse special build check",
|
|
"Docs Check",
|
|
"Fast test",
|
|
"Stateful tests (release)",
|
|
"Stateless tests (release)",
|
|
"Style Check",
|
|
"Unit tests (asan)",
|
|
"Unit tests (msan)",
|
|
"Unit tests (release-clang)",
|
|
"Unit tests (tsan)",
|
|
"Unit tests (ubsan)",
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class CheckDescription:
|
|
name: str
|
|
description: str # the check descriptions, will be put into the status table
|
|
match_func: Callable[[str], bool] # the function to check vs the commit status
|
|
|
|
def __hash__(self) -> int:
|
|
return hash(self.name + self.description)
|
|
|
|
|
|
CHECK_DESCRIPTIONS = [
|
|
CheckDescription(
|
|
"AST fuzzer",
|
|
"Runs randomly generated queries to catch program errors. "
|
|
"The build type is optionally given in parenthesis. "
|
|
"If it fails, ask a maintainer for help",
|
|
lambda x: x.startswith("AST fuzzer"),
|
|
),
|
|
CheckDescription(
|
|
"Bugfix validate check",
|
|
"Checks that either a new test (functional or integration) or there "
|
|
"some changed tests that fail with the binary built on master branch",
|
|
lambda x: x == "Bugfix validate check",
|
|
),
|
|
CheckDescription(
|
|
"CI running",
|
|
"A meta-check that indicates the running CI. Normally, it's in <b>success</b> or "
|
|
"<b>pending</b> state. The failed status indicates some problems with the PR",
|
|
lambda x: x == "CI running",
|
|
),
|
|
CheckDescription(
|
|
"ClickHouse build check",
|
|
"Builds ClickHouse in various configurations for use in further steps. "
|
|
"You have to fix the builds that fail. Build logs often has enough "
|
|
"information to fix the error, but you might have to reproduce the failure "
|
|
"locally. The <b>cmake</b> options can be found in the build log, grepping for "
|
|
'<b>cmake</b>. Use these options and follow the <a href="'
|
|
'https://clickhouse.com/docs/en/development/build">general build process</a>',
|
|
lambda x: x.startswith("ClickHouse") and x.endswith("build check"),
|
|
),
|
|
CheckDescription(
|
|
"Compatibility check",
|
|
"Checks that <b>clickhouse</b> binary runs on distributions with old libc "
|
|
"versions. If it fails, ask a maintainer for help",
|
|
lambda x: x.startswith("Compatibility check"),
|
|
),
|
|
CheckDescription(
|
|
"Docker image for servers",
|
|
"The check to build and optionally push the mentioned image to docker hub",
|
|
lambda x: x.startswith("Docker image")
|
|
and (x.endswith("building check") or x.endswith("build and push")),
|
|
),
|
|
CheckDescription(
|
|
"Docs Check", "Builds and tests the documentation", lambda x: x == "Docs Check"
|
|
),
|
|
CheckDescription(
|
|
"Fast test",
|
|
"Normally this is the first check that is ran for a PR. It builds ClickHouse "
|
|
'and runs most of <a href="https://clickhouse.com/docs/en/development/tests'
|
|
'#functional-tests">stateless functional tests</a>, '
|
|
"omitting some. If it fails, further checks are not started until it is fixed. "
|
|
"Look at the report to see which tests fail, then reproduce the failure "
|
|
'locally as described <a href="https://clickhouse.com/docs/en/development/'
|
|
'tests#functional-test-locally">here</a>',
|
|
lambda x: x == "Fast test",
|
|
),
|
|
CheckDescription(
|
|
"Flaky tests",
|
|
"Checks if new added or modified tests are flaky by running them repeatedly, "
|
|
"in parallel, with more randomization. Functional tests are run 100 times "
|
|
"with address sanitizer, and additional randomization of thread scheduling. "
|
|
"Integrational tests are run up to 10 times. If at least once a new test has "
|
|
"failed, or was too long, this check will be red. We don't allow flaky tests, "
|
|
'read <a href="https://clickhouse.com/blog/decorating-a-christmas-tree-with-'
|
|
'the-help-of-flaky-tests/">the doc</a>',
|
|
lambda x: "tests flaky check" in x,
|
|
),
|
|
CheckDescription(
|
|
"Install packages",
|
|
"Checks that the built packages are installable in a clear environment",
|
|
lambda x: x.startswith("Install packages ("),
|
|
),
|
|
CheckDescription(
|
|
"Integration tests",
|
|
"The integration tests report. In parenthesis the package type is given, "
|
|
"and in square brackets are the optional part/total tests",
|
|
lambda x: x.startswith("Integration tests ("),
|
|
),
|
|
CheckDescription(
|
|
"Mergeable Check",
|
|
"Checks if all other necessary checks are successful",
|
|
lambda x: x == "Mergeable Check",
|
|
),
|
|
CheckDescription(
|
|
"Performance Comparison",
|
|
"Measure changes in query performance. The performance test report is "
|
|
'described in detail <a href="https://github.com/ClickHouse/ClickHouse/tree'
|
|
'/master/docker/test/performance-comparison#how-to-read-the-report">here</a>. '
|
|
"In square brackets are the optional part/total tests",
|
|
lambda x: x.startswith("Performance Comparison"),
|
|
),
|
|
CheckDescription(
|
|
"Push to Dockerhub",
|
|
"The check for building and pushing the CI related docker images to docker hub",
|
|
lambda x: x.startswith("Push") and "to Dockerhub" in x,
|
|
),
|
|
CheckDescription(
|
|
"Sqllogic",
|
|
"Run clickhouse on the "
|
|
'<a href="https://www.sqlite.org/sqllogictest">sqllogic</a> '
|
|
"test set against sqlite and checks that all statements are passed",
|
|
lambda x: x.startswith("Sqllogic test"),
|
|
),
|
|
CheckDescription(
|
|
"SQLancer",
|
|
"Fuzzing tests that detect logical bugs with "
|
|
'<a href="https://github.com/sqlancer/sqlancer">SQLancer</a> tool',
|
|
lambda x: x.startswith("SQLancer"),
|
|
),
|
|
CheckDescription(
|
|
"Stateful tests",
|
|
"Runs stateful functional tests for ClickHouse binaries built in various "
|
|
"configurations -- release, debug, with sanitizers, etc",
|
|
lambda x: x.startswith("Stateful tests ("),
|
|
),
|
|
CheckDescription(
|
|
"Stateless tests",
|
|
"Runs stateless functional tests for ClickHouse binaries built in various "
|
|
"configurations -- release, debug, with sanitizers, etc",
|
|
lambda x: x.startswith("Stateless tests ("),
|
|
),
|
|
CheckDescription(
|
|
"Stress test",
|
|
"Runs stateless functional tests concurrently from several clients to detect "
|
|
"concurrency-related errors",
|
|
lambda x: x.startswith("Stress test ("),
|
|
),
|
|
CheckDescription(
|
|
"Style Check",
|
|
"Runs a set of checks to keep the code style clean. If some of tests failed, "
|
|
"see the related log from the report",
|
|
lambda x: x == "Style Check",
|
|
),
|
|
CheckDescription(
|
|
"Unit tests",
|
|
"Runs the unit tests for different release types",
|
|
lambda x: x.startswith("Unit tests ("),
|
|
),
|
|
CheckDescription(
|
|
"Upgrade check",
|
|
"Runs stress tests on server version from last release and then tries to "
|
|
"upgrade it to the version from the PR. It checks if the new server can "
|
|
"successfully startup without any errors, crashes or sanitizer asserts",
|
|
lambda x: x.startswith("Upgrade check ("),
|
|
),
|
|
CheckDescription(
|
|
"Falback for unknown",
|
|
"There's no description for the check yet, please add it to "
|
|
"tests/ci/ci_config.py:CHECK_DESCRIPTIONS",
|
|
lambda x: True,
|
|
),
|
|
]
|