ClickHouse/tests/ci/ci_config.py

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

470 lines
19 KiB
Python
Raw Normal View History

2021-11-12 11:12:42 +00:00
#!/usr/bin/env python3
import logging
from dataclasses import dataclass
from typing import Callable, Dict, List, Literal
@dataclass
class BuildConfig:
compiler: str
2023-09-03 18:05:54 +00:00
package_type: Literal["deb", "binary", "fuzzers"]
additional_pkgs: bool = False
debug_build: bool = False
sanitizer: str = ""
tidy: bool = False
comment: str = ""
static_binary_name: str = ""
@dataclass
class TestConfig:
required_build: str
force_tests: bool = False
2022-01-12 12:14:31 +00:00
BuildConfigs = Dict[str, BuildConfig]
BuildsReportConfig = Dict[str, List[str]]
TestConfigs = Dict[str, TestConfig]
2022-01-12 12:14:31 +00:00
@dataclass
class CiConfig:
build_config: BuildConfigs
builds_report_config: BuildsReportConfig
test_configs: TestConfigs
def validate(self) -> None:
errors = []
# All build configs must belong to build_report_config
for build_name in self.build_config.keys():
build_in_reports = False
for report_config in self.builds_report_config.values():
if build_name in report_config:
build_in_reports = True
break
if not build_in_reports:
logging.error(
"Build name %s does not belong to build reports", build_name
)
errors.append(
f"Build name {build_name} does not belong to build reports"
)
# And otherwise
for build_report_name, build_names in self.builds_report_config.items():
missed_names = [
name for name in build_names if name not in self.build_config.keys()
]
if missed_names:
logging.error(
"The following names of the build report '%s' "
"are missed in build_config: %s",
build_report_name,
missed_names,
)
errors.append(
f"The following names of the build report '{build_report_name}' "
f"are missed in build_config: {missed_names}",
)
# And finally, all of tests' requirements must be in the builds
for test_name, test_config in self.test_configs.items():
if test_config.required_build not in self.build_config.keys():
logging.error(
"The requierment '%s' for '%s' is not found in builds",
test_config,
test_name,
)
errors.append(
f"The requierment '{test_config}' for "
f"'{test_name}' is not found in builds"
)
if errors:
raise KeyError("config contains errors", errors)
CI_CONFIG = CiConfig(
build_config={
"package_release": BuildConfig(
compiler="clang-17",
package_type="deb",
static_binary_name="amd64",
additional_pkgs=True,
),
"package_aarch64": BuildConfig(
compiler="clang-17-aarch64",
package_type="deb",
static_binary_name="aarch64",
additional_pkgs=True,
),
"package_asan": BuildConfig(
compiler="clang-17",
sanitizer="address",
package_type="deb",
),
"package_ubsan": BuildConfig(
compiler="clang-17",
sanitizer="undefined",
package_type="deb",
),
"package_tsan": BuildConfig(
compiler="clang-17",
sanitizer="thread",
package_type="deb",
),
"package_msan": BuildConfig(
compiler="clang-17",
sanitizer="memory",
package_type="deb",
),
"package_debug": BuildConfig(
compiler="clang-17",
debug_build=True,
package_type="deb",
comment="Note: sparse checkout was used",
),
"binary_release": BuildConfig(
compiler="clang-17",
package_type="binary",
),
"binary_tidy": BuildConfig(
compiler="clang-17",
debug_build=True,
package_type="binary",
static_binary_name="debug-amd64",
tidy=True,
comment="clang-tidy is used for static analysis",
),
"binary_darwin": BuildConfig(
compiler="clang-17-darwin",
package_type="binary",
static_binary_name="macos",
),
"binary_aarch64": BuildConfig(
compiler="clang-17-aarch64",
package_type="binary",
),
"binary_aarch64_v80compat": BuildConfig(
compiler="clang-17-aarch64-v80compat",
package_type="binary",
static_binary_name="aarch64v80compat",
comment="For ARMv8.1 and older",
),
"binary_freebsd": BuildConfig(
compiler="clang-17-freebsd",
package_type="binary",
static_binary_name="freebsd",
),
"binary_darwin_aarch64": BuildConfig(
compiler="clang-17-darwin-aarch64",
package_type="binary",
static_binary_name="macos-aarch64",
),
"binary_ppc64le": BuildConfig(
compiler="clang-17-ppc64le",
package_type="binary",
static_binary_name="powerpc64le",
),
"binary_amd64_compat": BuildConfig(
compiler="clang-17-amd64-compat",
package_type="binary",
static_binary_name="amd64compat",
comment="SSE2-only build",
),
"binary_riscv64": BuildConfig(
compiler="clang-17-riscv64",
package_type="binary",
static_binary_name="riscv64",
),
2023-08-09 00:44:01 +00:00
"binary_s390x": BuildConfig(
compiler="clang-17-s390x",
2023-08-09 00:44:01 +00:00
package_type="binary",
static_binary_name="s390x",
),
"fuzzers": BuildConfig(
compiler="clang-16",
package_type="fuzzers",
),
},
builds_report_config={
2022-07-21 10:27:45 +00:00
"ClickHouse build check": [
2021-11-30 15:33:29 +00:00
"package_release",
2021-12-08 21:52:00 +00:00
"package_aarch64",
2021-11-30 15:33:29 +00:00
"package_asan",
"package_ubsan",
"package_tsan",
"package_msan",
"package_debug",
2021-12-14 17:47:45 +00:00
"binary_release",
2023-09-03 17:38:36 +00:00
"fuzzers",
2021-11-30 15:33:29 +00:00
],
2022-07-21 10:27:45 +00:00
"ClickHouse special build check": [
2021-11-30 15:33:29 +00:00
"binary_tidy",
"binary_darwin",
2021-12-15 07:45:27 +00:00
"binary_aarch64",
2022-09-21 13:09:24 +00:00
"binary_aarch64_v80compat",
2021-11-30 15:33:29 +00:00
"binary_freebsd",
2021-12-15 07:45:27 +00:00
"binary_darwin_aarch64",
"binary_ppc64le",
2022-08-14 04:14:43 +00:00
"binary_riscv64",
2023-08-09 00:44:01 +00:00
"binary_s390x",
"binary_amd64_compat",
2021-11-30 15:33:29 +00:00
],
},
test_configs={
"Install packages (amd64)": TestConfig("package_release"),
"Install packages (arm64)": TestConfig("package_aarch64"),
"Stateful tests (asan)": TestConfig("package_asan"),
"Stateful tests (tsan)": TestConfig("package_tsan"),
"Stateful tests (msan)": TestConfig("package_msan"),
"Stateful tests (ubsan)": TestConfig("package_ubsan"),
"Stateful tests (debug)": TestConfig("package_debug"),
"Stateful tests (release)": TestConfig("package_release"),
"Stateful tests (aarch64)": TestConfig("package_aarch64"),
"Stateful tests (release, DatabaseOrdinary)": TestConfig("package_release"),
"Stateful tests (release, DatabaseReplicated)": TestConfig("package_release"),
2023-02-03 13:34:18 +00:00
# Stateful tests for parallel replicas
"Stateful tests (release, ParallelReplicas)": TestConfig("package_release"),
"Stateful tests (debug, ParallelReplicas)": TestConfig("package_debug"),
"Stateful tests (asan, ParallelReplicas)": TestConfig("package_asan"),
"Stateful tests (msan, ParallelReplicas)": TestConfig("package_msan"),
"Stateful tests (ubsan, ParallelReplicas)": TestConfig("package_ubsan"),
"Stateful tests (tsan, ParallelReplicas)": TestConfig("package_tsan"),
2023-02-03 13:34:18 +00:00
# End stateful tests for parallel replicas
"Stateless tests (asan)": TestConfig("package_asan"),
"Stateless tests (tsan)": TestConfig("package_tsan"),
"Stateless tests (msan)": TestConfig("package_msan"),
"Stateless tests (ubsan)": TestConfig("package_ubsan"),
"Stateless tests (debug)": TestConfig("package_debug"),
"Stateless tests (release)": TestConfig("package_release"),
"Stateless tests (aarch64)": TestConfig("package_aarch64"),
"Stateless tests (release, wide parts enabled)": TestConfig("package_release"),
"Stateless tests (release, analyzer)": TestConfig("package_release"),
"Stateless tests (release, DatabaseOrdinary)": TestConfig("package_release"),
"Stateless tests (release, DatabaseReplicated)": TestConfig("package_release"),
"Stateless tests (release, s3 storage)": TestConfig("package_release"),
"Stateless tests (debug, s3 storage)": TestConfig("package_debug"),
"Stateless tests (tsan, s3 storage)": TestConfig("package_tsan"),
"Stress test (asan)": TestConfig("package_asan"),
"Stress test (tsan)": TestConfig("package_tsan"),
"Stress test (ubsan)": TestConfig("package_ubsan"),
"Stress test (msan)": TestConfig("package_msan"),
"Stress test (debug)": TestConfig("package_debug"),
"Upgrade check (asan)": TestConfig("package_asan"),
"Upgrade check (tsan)": TestConfig("package_tsan"),
"Upgrade check (msan)": TestConfig("package_msan"),
"Upgrade check (debug)": TestConfig("package_debug"),
"Integration tests (asan)": TestConfig("package_asan"),
"Integration tests (asan, analyzer)": TestConfig("package_asan"),
"Integration tests (tsan)": TestConfig("package_tsan"),
"Integration tests (release)": TestConfig("package_release"),
"Integration tests (msan)": TestConfig("package_msan"),
"Integration tests flaky check (asan)": TestConfig("package_asan"),
"Compatibility check (amd64)": TestConfig("package_release"),
"Compatibility check (aarch64)": TestConfig("package_aarch64"),
"Unit tests (release)": TestConfig("binary_release"),
"Unit tests (asan)": TestConfig("package_asan"),
"Unit tests (msan)": TestConfig("package_msan"),
"Unit tests (tsan)": TestConfig("package_tsan"),
"Unit tests (ubsan)": TestConfig("package_ubsan"),
"AST fuzzer (debug)": TestConfig("package_debug"),
"AST fuzzer (asan)": TestConfig("package_asan"),
"AST fuzzer (msan)": TestConfig("package_msan"),
"AST fuzzer (tsan)": TestConfig("package_tsan"),
"AST fuzzer (ubsan)": TestConfig("package_ubsan"),
"Stateless tests flaky check (asan)": TestConfig("package_asan"),
"ClickHouse Keeper Jepsen": TestConfig("binary_release"),
"ClickHouse Server Jepsen": TestConfig("binary_release"),
"Performance Comparison": TestConfig("package_release"),
"Performance Comparison Aarch64": TestConfig("package_aarch64"),
"SQLancer (release)": TestConfig("package_release"),
"SQLancer (debug)": TestConfig("package_debug"),
"Sqllogic test (release)": TestConfig("package_release"),
2023-08-04 19:16:56 +00:00
"SQLTest": TestConfig("package_release"),
"libFuzzer tests": TestConfig("fuzzers"),
2021-12-14 17:47:45 +00:00
},
)
CI_CONFIG.validate()
# checks required by Mergeable Check
REQUIRED_CHECKS = [
"ClickHouse build check",
"ClickHouse special build check",
"Docs Check",
"Fast test",
"Stateful tests (release)",
"Stateless tests (release)",
"Style Check",
2022-08-15 23:05:40 +00:00
"Unit tests (asan)",
"Unit tests (msan)",
2023-07-31 03:23:28 +00:00
"Unit tests (release)",
2022-08-15 23:05:40 +00:00
"Unit tests (tsan)",
"Unit tests (ubsan)",
]
@dataclass
class CheckDescription:
name: str
description: str # the check descriptions, will be put into the status table
match_func: Callable[[str], bool] # the function to check vs the commit status
def __hash__(self) -> int:
return hash(self.name + self.description)
CHECK_DESCRIPTIONS = [
CheckDescription(
"AST fuzzer",
"Runs randomly generated queries to catch program errors. "
"The build type is optionally given in parenthesis. "
"If it fails, ask a maintainer for help",
lambda x: x.startswith("AST fuzzer"),
),
CheckDescription(
"Bugfix validate check",
"Checks that either a new test (functional or integration) or there "
"some changed tests that fail with the binary built on master branch",
lambda x: x == "Bugfix validate check",
),
2023-04-20 11:55:33 +00:00
CheckDescription(
"CI running",
"A meta-check that indicates the running CI. Normally, it's in <b>success</b> or "
"<b>pending</b> state. The failed status indicates some problems with the PR",
2023-04-20 11:55:33 +00:00
lambda x: x == "CI running",
),
CheckDescription(
"ClickHouse build check",
"Builds ClickHouse in various configurations for use in further steps. "
"You have to fix the builds that fail. Build logs often has enough "
"information to fix the error, but you might have to reproduce the failure "
"locally. The <b>cmake</b> options can be found in the build log, grepping for "
'<b>cmake</b>. Use these options and follow the <a href="'
'https://clickhouse.com/docs/en/development/build">general build process</a>',
lambda x: x.startswith("ClickHouse") and x.endswith("build check"),
),
CheckDescription(
"Compatibility check",
"Checks that <b>clickhouse</b> binary runs on distributions with old libc "
"versions. If it fails, ask a maintainer for help",
lambda x: x.startswith("Compatibility check"),
),
CheckDescription(
"Docker image for servers",
"The check to build and optionally push the mentioned image to docker hub",
lambda x: x.startswith("Docker image")
and (x.endswith("building check") or x.endswith("build and push")),
),
CheckDescription(
"Docs Check", "Builds and tests the documentation", lambda x: x == "Docs Check"
),
CheckDescription(
"Fast test",
"Normally this is the first check that is ran for a PR. It builds ClickHouse "
'and runs most of <a href="https://clickhouse.com/docs/en/development/tests'
'#functional-tests">stateless functional tests</a>, '
"omitting some. If it fails, further checks are not started until it is fixed. "
"Look at the report to see which tests fail, then reproduce the failure "
'locally as described <a href="https://clickhouse.com/docs/en/development/'
'tests#functional-test-locally">here</a>',
lambda x: x == "Fast test",
),
CheckDescription(
"Flaky tests",
"Checks if new added or modified tests are flaky by running them repeatedly, "
"in parallel, with more randomization. Functional tests are run 100 times "
"with address sanitizer, and additional randomization of thread scheduling. "
"Integrational tests are run up to 10 times. If at least once a new test has "
"failed, or was too long, this check will be red. We don't allow flaky tests, "
'read <a href="https://clickhouse.com/blog/decorating-a-christmas-tree-with-'
'the-help-of-flaky-tests/">the doc</a>',
lambda x: "tests flaky check" in x,
),
CheckDescription(
"Install packages",
"Checks that the built packages are installable in a clear environment",
lambda x: x.startswith("Install packages ("),
),
CheckDescription(
"Integration tests",
"The integration tests report. In parenthesis the package type is given, "
"and in square brackets are the optional part/total tests",
lambda x: x.startswith("Integration tests ("),
),
CheckDescription(
"Mergeable Check",
"Checks if all other necessary checks are successful",
lambda x: x == "Mergeable Check",
),
CheckDescription(
"Performance Comparison",
"Measure changes in query performance. The performance test report is "
'described in detail <a href="https://github.com/ClickHouse/ClickHouse/tree'
'/master/docker/test/performance-comparison#how-to-read-the-report">here</a>. '
"In square brackets are the optional part/total tests",
lambda x: x.startswith("Performance Comparison"),
),
CheckDescription(
"Push to Dockerhub",
"The check for building and pushing the CI related docker images to docker hub",
lambda x: x.startswith("Push") and "to Dockerhub" in x,
),
CheckDescription(
"Sqllogic",
"Run clickhouse on the "
'<a href="https://www.sqlite.org/sqllogictest">sqllogic</a> '
"test set against sqlite and checks that all statements are passed",
lambda x: x.startswith("Sqllogic test"),
),
CheckDescription(
"SQLancer",
"Fuzzing tests that detect logical bugs with "
'<a href="https://github.com/sqlancer/sqlancer">SQLancer</a> tool',
lambda x: x.startswith("SQLancer"),
),
CheckDescription(
"Stateful tests",
"Runs stateful functional tests for ClickHouse binaries built in various "
"configurations -- release, debug, with sanitizers, etc",
lambda x: x.startswith("Stateful tests ("),
),
CheckDescription(
"Stateless tests",
"Runs stateless functional tests for ClickHouse binaries built in various "
"configurations -- release, debug, with sanitizers, etc",
lambda x: x.startswith("Stateless tests ("),
),
CheckDescription(
"Stress test",
"Runs stateless functional tests concurrently from several clients to detect "
"concurrency-related errors",
lambda x: x.startswith("Stress test ("),
),
CheckDescription(
"Style Check",
"Runs a set of checks to keep the code style clean. If some of tests failed, "
"see the related log from the report",
lambda x: x == "Style Check",
),
CheckDescription(
"Unit tests",
"Runs the unit tests for different release types",
lambda x: x.startswith("Unit tests ("),
),
CheckDescription(
"Upgrade check",
"Runs stress tests on server version from last release and then tries to "
"upgrade it to the version from the PR. It checks if the new server can "
"successfully startup without any errors, crashes or sanitizer asserts",
lambda x: x.startswith("Upgrade check ("),
),
CheckDescription(
"Falback for unknown",
"There's no description for the check yet, please add it to "
"tests/ci/ci_config.py:CHECK_DESCRIPTIONS",
lambda x: True,
),
]