mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 00:30:49 +00:00
Refactor CI_CONFIG from dict to dataclasses
This commit is contained in:
Mikhail f. Shiryaev
parent
6dcdc4cd38
commit
8499956321
@ -38,11 +38,11 @@ BUILD_LOG_NAME = "build_log.log"
|
||||
|
||||
|
||||
def _can_export_binaries(build_config: BuildConfig) -> bool:
|
||||
if build_config["package_type"] != "deb":
|
||||
if build_config.package_type != "deb":
|
||||
return False
|
||||
if build_config["sanitizer"] != "":
|
||||
if build_config.sanitizer != "":
|
||||
return True
|
||||
if build_config["debug_build"]:
|
||||
if build_config.debug_build:
|
||||
return True
|
||||
return False
|
||||
|
||||
@ -55,26 +55,26 @@ def get_packager_cmd(
|
||||
image_version: str,
|
||||
official: bool,
|
||||
) -> str:
|
||||
package_type = build_config["package_type"]
|
||||
comp = build_config["compiler"]
|
||||
package_type = build_config.package_type
|
||||
comp = build_config.compiler
|
||||
cmake_flags = "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=1"
|
||||
cmd = (
|
||||
f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} "
|
||||
f"--package-type={package_type} --compiler={comp}"
|
||||
)
|
||||
|
||||
if build_config["debug_build"]:
|
||||
if build_config.debug_build:
|
||||
cmd += " --debug-build"
|
||||
if build_config["sanitizer"]:
|
||||
cmd += f" --sanitizer={build_config['sanitizer']}"
|
||||
if build_config["tidy"] == "enable":
|
||||
if build_config.sanitizer:
|
||||
cmd += f" --sanitizer={build_config.sanitizer}"
|
||||
if build_config.tidy:
|
||||
cmd += " --clang-tidy"
|
||||
|
||||
cmd += " --cache=sccache"
|
||||
cmd += " --s3-rw-access"
|
||||
cmd += f" --s3-bucket={S3_BUILDS_BUCKET}"
|
||||
|
||||
if "additional_pkgs" in build_config and build_config["additional_pkgs"]:
|
||||
if build_config.additional_pkgs:
|
||||
cmd += " --additional-pkgs"
|
||||
|
||||
cmd += f" --docker-image-version={image_version}"
|
||||
@ -180,7 +180,7 @@ def create_json_artifact(
|
||||
result = {
|
||||
"log_url": log_url,
|
||||
"build_urls": build_urls,
|
||||
"build_config": build_config,
|
||||
"build_config": build_config.__dict__,
|
||||
"elapsed_seconds": elapsed,
|
||||
"status": success,
|
||||
"job_name": GITHUB_JOB,
|
||||
@ -220,7 +220,7 @@ def upload_master_static_binaries(
|
||||
build_output_path: str,
|
||||
) -> None:
|
||||
"""Upload binary artifacts to a static S3 links"""
|
||||
static_binary_name = build_config.get("static_binary_name", False)
|
||||
static_binary_name = build_config.static_binary_name
|
||||
if pr_info.number != 0:
|
||||
return
|
||||
elif not static_binary_name:
|
||||
@ -240,7 +240,7 @@ def main():
|
||||
stopwatch = Stopwatch()
|
||||
build_name = sys.argv[1]
|
||||
|
||||
build_config = CI_CONFIG["build_config"][build_name]
|
||||
build_config = CI_CONFIG.build_config[build_name]
|
||||
|
||||
if not os.path.exists(TEMP_PATH):
|
||||
os.makedirs(TEMP_PATH)
|
||||
@ -270,8 +270,6 @@ def main():
|
||||
logging.info("Got version from repo %s", version.string)
|
||||
|
||||
official_flag = pr_info.number == 0
|
||||
if "official" in build_config:
|
||||
official_flag = build_config["official"]
|
||||
|
||||
version_type = "testing"
|
||||
if "release" in pr_info.labels or "release-lts" in pr_info.labels:
|
||||
|
||||
@ -91,7 +91,7 @@ def get_gh_api(
|
||||
|
||||
|
||||
def get_build_name_for_check(check_name: str) -> str:
|
||||
return CI_CONFIG["tests_config"][check_name]["required_build"] # type: ignore
|
||||
return CI_CONFIG.test_configs[check_name].required_build
|
||||
|
||||
|
||||
def read_build_urls(build_name: str, reports_path: str) -> List[str]:
|
||||
|
||||
@ -149,7 +149,7 @@ def main():
|
||||
logging.info("Check is already finished according to github status, exiting")
|
||||
sys.exit(0)
|
||||
|
||||
builds_for_check = CI_CONFIG["builds_report_config"][build_check_name]
|
||||
builds_for_check = CI_CONFIG.builds_report_config[build_check_name]
|
||||
required_builds = required_builds or len(builds_for_check)
|
||||
|
||||
# Collect reports from json artifacts
|
||||
|
||||
@ -1,180 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import logging
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Dict, TypeVar
|
||||
from typing import Callable, Dict, List, Literal
|
||||
|
||||
ConfValue = TypeVar("ConfValue", str, bool)
|
||||
BuildConfig = Dict[str, ConfValue]
|
||||
|
||||
CI_CONFIG = {
|
||||
"build_config": {
|
||||
"package_release": {
|
||||
"compiler": "clang-16",
|
||||
"debug_build": False,
|
||||
"sanitizer": "",
|
||||
"package_type": "deb",
|
||||
"static_binary_name": "amd64",
|
||||
"additional_pkgs": True,
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "",
|
||||
},
|
||||
"package_aarch64": {
|
||||
"compiler": "clang-16-aarch64",
|
||||
"debug_build": False,
|
||||
"sanitizer": "",
|
||||
"package_type": "deb",
|
||||
"static_binary_name": "aarch64",
|
||||
"additional_pkgs": True,
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "",
|
||||
},
|
||||
"package_asan": {
|
||||
"compiler": "clang-16",
|
||||
"debug_build": False,
|
||||
"sanitizer": "address",
|
||||
"package_type": "deb",
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "",
|
||||
},
|
||||
"package_ubsan": {
|
||||
"compiler": "clang-16",
|
||||
"debug_build": False,
|
||||
"sanitizer": "undefined",
|
||||
"package_type": "deb",
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "",
|
||||
},
|
||||
"package_tsan": {
|
||||
"compiler": "clang-16",
|
||||
"debug_build": False,
|
||||
"sanitizer": "thread",
|
||||
"package_type": "deb",
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "",
|
||||
},
|
||||
"package_msan": {
|
||||
"compiler": "clang-16",
|
||||
"debug_build": False,
|
||||
"sanitizer": "memory",
|
||||
"package_type": "deb",
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "",
|
||||
},
|
||||
"package_debug": {
|
||||
"compiler": "clang-16",
|
||||
"debug_build": True,
|
||||
"sanitizer": "",
|
||||
"package_type": "deb",
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "Note: sparse checkout was used",
|
||||
},
|
||||
"binary_release": {
|
||||
"compiler": "clang-16",
|
||||
"debug_build": False,
|
||||
"sanitizer": "",
|
||||
"package_type": "binary",
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "",
|
||||
},
|
||||
"binary_tidy": {
|
||||
"compiler": "clang-16",
|
||||
"debug_build": True,
|
||||
"sanitizer": "",
|
||||
"package_type": "binary",
|
||||
"static_binary_name": "debug-amd64",
|
||||
"tidy": "enable",
|
||||
"with_coverage": False,
|
||||
"comment": "clang-tidy is used for static analysis",
|
||||
},
|
||||
"binary_darwin": {
|
||||
"compiler": "clang-16-darwin",
|
||||
"debug_build": False,
|
||||
"sanitizer": "",
|
||||
"package_type": "binary",
|
||||
"static_binary_name": "macos",
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "",
|
||||
},
|
||||
"binary_aarch64": {
|
||||
"compiler": "clang-16-aarch64",
|
||||
"debug_build": False,
|
||||
"sanitizer": "",
|
||||
"package_type": "binary",
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "",
|
||||
},
|
||||
"binary_aarch64_v80compat": {
|
||||
"compiler": "clang-16-aarch64-v80compat",
|
||||
"debug_build": False,
|
||||
"sanitizer": "",
|
||||
"package_type": "binary",
|
||||
"static_binary_name": "aarch64v80compat",
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "For ARMv8.1 and older",
|
||||
},
|
||||
"binary_freebsd": {
|
||||
"compiler": "clang-16-freebsd",
|
||||
"debug_build": False,
|
||||
"sanitizer": "",
|
||||
"package_type": "binary",
|
||||
"static_binary_name": "freebsd",
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "",
|
||||
},
|
||||
"binary_darwin_aarch64": {
|
||||
"compiler": "clang-16-darwin-aarch64",
|
||||
"debug_build": False,
|
||||
"sanitizer": "",
|
||||
"package_type": "binary",
|
||||
"static_binary_name": "macos-aarch64",
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "",
|
||||
},
|
||||
"binary_ppc64le": {
|
||||
"compiler": "clang-16-ppc64le",
|
||||
"debug_build": False,
|
||||
"sanitizer": "",
|
||||
"package_type": "binary",
|
||||
"static_binary_name": "powerpc64le",
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "",
|
||||
},
|
||||
"binary_amd64_compat": {
|
||||
"compiler": "clang-16-amd64-compat",
|
||||
"debug_build": False,
|
||||
"sanitizer": "",
|
||||
"package_type": "binary",
|
||||
"static_binary_name": "amd64compat",
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "SSE2-only build",
|
||||
},
|
||||
"binary_riscv64": {
|
||||
"compiler": "clang-16-riscv64",
|
||||
"debug_build": False,
|
||||
"sanitizer": "",
|
||||
"package_type": "binary",
|
||||
"static_binary_name": "riscv64",
|
||||
"tidy": "disable",
|
||||
"with_coverage": False,
|
||||
"comment": "",
|
||||
},
|
||||
@dataclass
|
||||
class BuildConfig:
|
||||
compiler: str
|
||||
package_type: Literal["deb", "binary"]
|
||||
additional_pkgs: bool = False
|
||||
debug_build: bool = False
|
||||
sanitizer: str = ""
|
||||
tidy: bool = False
|
||||
comment: str = ""
|
||||
static_binary_name: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestConfig:
|
||||
required_build: str
|
||||
force_tests: bool = False
|
||||
|
||||
|
||||
BuildConfigs = Dict[str, BuildConfig]
|
||||
BuildsReportConfig = Dict[str, List[str]]
|
||||
TestConfigs = Dict[str, TestConfig]
|
||||
|
||||
|
||||
@dataclass
|
||||
class CiConfig:
|
||||
build_config: BuildConfigs
|
||||
builds_report_config: BuildsReportConfig
|
||||
test_configs: TestConfigs
|
||||
|
||||
def validate(self) -> None:
|
||||
errors = []
|
||||
# All build configs must belong to build_report_config
|
||||
for build_name in self.build_config.keys():
|
||||
build_in_reports = False
|
||||
for report_config in self.builds_report_config.values():
|
||||
if build_name in report_config:
|
||||
build_in_reports = True
|
||||
break
|
||||
if not build_in_reports:
|
||||
logging.error(
|
||||
"Build name %s does not belong to build reports", build_name
|
||||
)
|
||||
errors.append(
|
||||
f"Build name {build_name} does not belong to build reports"
|
||||
)
|
||||
# And otherwise
|
||||
for build_report_name, build_names in self.builds_report_config.items():
|
||||
missed_names = [
|
||||
name for name in build_names if name not in self.build_config.keys()
|
||||
]
|
||||
if missed_names:
|
||||
logging.error(
|
||||
"The following names of the build report '%s' "
|
||||
"are missed in build_config: %s",
|
||||
build_report_name,
|
||||
missed_names,
|
||||
)
|
||||
errors.append(
|
||||
f"The following names of the build report '{build_report_name}' "
|
||||
f"are missed in build_config: {missed_names}",
|
||||
)
|
||||
# And finally, all of tests' requirements must be in the builds
|
||||
for test_name, test_config in self.test_configs.items():
|
||||
if test_config.required_build not in self.build_config.keys():
|
||||
logging.error(
|
||||
"The requierment '%s' for '%s' is not found in builds",
|
||||
test_config,
|
||||
test_name,
|
||||
)
|
||||
errors.append(
|
||||
f"The requierment '{test_config}' for "
|
||||
f"'{test_name}' is not found in builds"
|
||||
)
|
||||
|
||||
if errors:
|
||||
raise KeyError("config contains errors", errors)
|
||||
|
||||
|
||||
CI_CONFIG = CiConfig(
|
||||
build_config={
|
||||
"package_release": BuildConfig(
|
||||
compiler="clang-16",
|
||||
package_type="deb",
|
||||
static_binary_name="amd64",
|
||||
additional_pkgs=True,
|
||||
),
|
||||
"package_aarch64": BuildConfig(
|
||||
compiler="clang-16-aarch64",
|
||||
package_type="deb",
|
||||
static_binary_name="aarch64",
|
||||
additional_pkgs=True,
|
||||
),
|
||||
"package_asan": BuildConfig(
|
||||
compiler="clang-16",
|
||||
sanitizer="address",
|
||||
package_type="deb",
|
||||
),
|
||||
"package_ubsan": BuildConfig(
|
||||
compiler="clang-16",
|
||||
sanitizer="undefined",
|
||||
package_type="deb",
|
||||
),
|
||||
"package_tsan": BuildConfig(
|
||||
compiler="clang-16",
|
||||
sanitizer="thread",
|
||||
package_type="deb",
|
||||
),
|
||||
"package_msan": BuildConfig(
|
||||
compiler="clang-16",
|
||||
sanitizer="memory",
|
||||
package_type="deb",
|
||||
),
|
||||
"package_debug": BuildConfig(
|
||||
compiler="clang-16",
|
||||
debug_build=True,
|
||||
package_type="deb",
|
||||
comment="Note: sparse checkout was used",
|
||||
),
|
||||
"binary_release": BuildConfig(
|
||||
compiler="clang-16",
|
||||
package_type="binary",
|
||||
),
|
||||
"binary_tidy": BuildConfig(
|
||||
compiler="clang-16",
|
||||
debug_build=True,
|
||||
package_type="binary",
|
||||
static_binary_name="debug-amd64",
|
||||
tidy=True,
|
||||
comment="clang-tidy is used for static analysis",
|
||||
),
|
||||
"binary_darwin": BuildConfig(
|
||||
compiler="clang-16-darwin",
|
||||
package_type="binary",
|
||||
static_binary_name="macos",
|
||||
),
|
||||
"binary_aarch64": BuildConfig(
|
||||
compiler="clang-16-aarch64",
|
||||
package_type="binary",
|
||||
),
|
||||
"binary_aarch64_v80compat": BuildConfig(
|
||||
compiler="clang-16-aarch64-v80compat",
|
||||
package_type="binary",
|
||||
static_binary_name="aarch64v80compat",
|
||||
comment="For ARMv8.1 and older",
|
||||
),
|
||||
"binary_freebsd": BuildConfig(
|
||||
compiler="clang-16-freebsd",
|
||||
package_type="binary",
|
||||
static_binary_name="freebsd",
|
||||
),
|
||||
"binary_darwin_aarch64": BuildConfig(
|
||||
compiler="clang-16-darwin-aarch64",
|
||||
package_type="binary",
|
||||
static_binary_name="macos-aarch64",
|
||||
),
|
||||
"binary_ppc64le": BuildConfig(
|
||||
compiler="clang-16-ppc64le",
|
||||
package_type="binary",
|
||||
static_binary_name="powerpc64le",
|
||||
),
|
||||
"binary_amd64_compat": BuildConfig(
|
||||
compiler="clang-16-amd64-compat",
|
||||
package_type="binary",
|
||||
static_binary_name="amd64compat",
|
||||
comment="SSE2-only build",
|
||||
),
|
||||
"binary_riscv64": BuildConfig(
|
||||
compiler="clang-16-riscv64",
|
||||
package_type="binary",
|
||||
static_binary_name="riscv64",
|
||||
),
|
||||
},
|
||||
"builds_report_config": {
|
||||
builds_report_config={
|
||||
"ClickHouse build check": [
|
||||
"package_release",
|
||||
"package_aarch64",
|
||||
@ -197,213 +201,79 @@ CI_CONFIG = {
|
||||
"binary_amd64_compat",
|
||||
],
|
||||
},
|
||||
"tests_config": {
|
||||
# required_build - build name for artifacts
|
||||
# force_tests - force success status for tests
|
||||
"Install packages (amd64)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
"Install packages (arm64)": {
|
||||
"required_build": "package_aarch64",
|
||||
},
|
||||
"Stateful tests (asan)": {
|
||||
"required_build": "package_asan",
|
||||
},
|
||||
"Stateful tests (tsan)": {
|
||||
"required_build": "package_tsan",
|
||||
},
|
||||
"Stateful tests (msan)": {
|
||||
"required_build": "package_msan",
|
||||
},
|
||||
"Stateful tests (ubsan)": {
|
||||
"required_build": "package_ubsan",
|
||||
},
|
||||
"Stateful tests (debug)": {
|
||||
"required_build": "package_debug",
|
||||
},
|
||||
"Stateful tests (release)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
"Stateful tests (aarch64)": {
|
||||
"required_build": "package_aarch64",
|
||||
},
|
||||
"Stateful tests (release, DatabaseOrdinary)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
"Stateful tests (release, DatabaseReplicated)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
test_configs={
|
||||
"Install packages (amd64)": TestConfig("package_release"),
|
||||
"Install packages (arm64)": TestConfig("package_aarch64"),
|
||||
"Stateful tests (asan)": TestConfig("package_asan"),
|
||||
"Stateful tests (tsan)": TestConfig("package_tsan"),
|
||||
"Stateful tests (msan)": TestConfig("package_msan"),
|
||||
"Stateful tests (ubsan)": TestConfig("package_ubsan"),
|
||||
"Stateful tests (debug)": TestConfig("package_debug"),
|
||||
"Stateful tests (release)": TestConfig("package_release"),
|
||||
"Stateful tests (aarch64)": TestConfig("package_aarch64"),
|
||||
"Stateful tests (release, DatabaseOrdinary)": TestConfig("package_release"),
|
||||
"Stateful tests (release, DatabaseReplicated)": TestConfig("package_release"),
|
||||
# Stateful tests for parallel replicas
|
||||
"Stateful tests (release, ParallelReplicas)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
"Stateful tests (debug, ParallelReplicas)": {
|
||||
"required_build": "package_debug",
|
||||
},
|
||||
"Stateful tests (asan, ParallelReplicas)": {
|
||||
"required_build": "package_asan",
|
||||
},
|
||||
"Stateful tests (msan, ParallelReplicas)": {
|
||||
"required_build": "package_msan",
|
||||
},
|
||||
"Stateful tests (ubsan, ParallelReplicas)": {
|
||||
"required_build": "package_ubsan",
|
||||
},
|
||||
"Stateful tests (tsan, ParallelReplicas)": {
|
||||
"required_build": "package_tsan",
|
||||
},
|
||||
"Stateful tests (release, ParallelReplicas)": TestConfig("package_release"),
|
||||
"Stateful tests (debug, ParallelReplicas)": TestConfig("package_debug"),
|
||||
"Stateful tests (asan, ParallelReplicas)": TestConfig("package_asan"),
|
||||
"Stateful tests (msan, ParallelReplicas)": TestConfig("package_msan"),
|
||||
"Stateful tests (ubsan, ParallelReplicas)": TestConfig("package_ubsan"),
|
||||
"Stateful tests (tsan, ParallelReplicas)": TestConfig("package_tsan"),
|
||||
# End stateful tests for parallel replicas
|
||||
"Stateless tests (asan)": {
|
||||
"required_build": "package_asan",
|
||||
},
|
||||
"Stateless tests (tsan)": {
|
||||
"required_build": "package_tsan",
|
||||
},
|
||||
"Stateless tests (msan)": {
|
||||
"required_build": "package_msan",
|
||||
},
|
||||
"Stateless tests (ubsan)": {
|
||||
"required_build": "package_ubsan",
|
||||
},
|
||||
"Stateless tests (debug)": {
|
||||
"required_build": "package_debug",
|
||||
},
|
||||
"Stateless tests (release)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
"Stateless tests (aarch64)": {
|
||||
"required_build": "package_aarch64",
|
||||
},
|
||||
"Stateless tests (release, wide parts enabled)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
"Stateless tests (release, analyzer)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
"Stateless tests (release, DatabaseOrdinary)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
"Stateless tests (release, DatabaseReplicated)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
"Stateless tests (release, s3 storage)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
"Stateless tests (debug, s3 storage)": {
|
||||
"required_build": "package_debug",
|
||||
},
|
||||
"Stateless tests (tsan, s3 storage)": {
|
||||
"required_build": "package_tsan",
|
||||
},
|
||||
"Stress test (asan)": {
|
||||
"required_build": "package_asan",
|
||||
},
|
||||
"Stress test (tsan)": {
|
||||
"required_build": "package_tsan",
|
||||
},
|
||||
"Stress test (ubsan)": {
|
||||
"required_build": "package_ubsan",
|
||||
},
|
||||
"Stress test (msan)": {
|
||||
"required_build": "package_msan",
|
||||
},
|
||||
"Stress test (debug)": {
|
||||
"required_build": "package_debug",
|
||||
},
|
||||
"Upgrade check (asan)": {
|
||||
"required_build": "package_asan",
|
||||
},
|
||||
"Upgrade check (tsan)": {
|
||||
"required_build": "package_tsan",
|
||||
},
|
||||
"Upgrade check (msan)": {
|
||||
"required_build": "package_msan",
|
||||
},
|
||||
"Upgrade check (debug)": {
|
||||
"required_build": "package_debug",
|
||||
},
|
||||
"Integration tests (asan)": {
|
||||
"required_build": "package_asan",
|
||||
},
|
||||
"Integration tests (asan, analyzer)": {
|
||||
"required_build": "package_asan",
|
||||
},
|
||||
"Integration tests (tsan)": {
|
||||
"required_build": "package_tsan",
|
||||
},
|
||||
"Integration tests (release)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
"Integration tests (msan)": {
|
||||
"required_build": "package_msan",
|
||||
},
|
||||
"Integration tests flaky check (asan)": {
|
||||
"required_build": "package_asan",
|
||||
},
|
||||
"Compatibility check (amd64)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
"Compatibility check (aarch64)": {
|
||||
"required_build": "package_aarch64",
|
||||
},
|
||||
"Unit tests (release)": {
|
||||
"required_build": "binary_release",
|
||||
},
|
||||
"Unit tests (asan)": {
|
||||
"required_build": "package_asan",
|
||||
},
|
||||
"Unit tests (msan)": {
|
||||
"required_build": "package_msan",
|
||||
},
|
||||
"Unit tests (tsan)": {
|
||||
"required_build": "package_tsan",
|
||||
},
|
||||
"Unit tests (ubsan)": {
|
||||
"required_build": "package_ubsan",
|
||||
},
|
||||
"AST fuzzer (debug)": {
|
||||
"required_build": "package_debug",
|
||||
},
|
||||
"AST fuzzer (asan)": {
|
||||
"required_build": "package_asan",
|
||||
},
|
||||
"AST fuzzer (msan)": {
|
||||
"required_build": "package_msan",
|
||||
},
|
||||
"AST fuzzer (tsan)": {
|
||||
"required_build": "package_tsan",
|
||||
},
|
||||
"AST fuzzer (ubsan)": {
|
||||
"required_build": "package_ubsan",
|
||||
},
|
||||
"Stateless tests flaky check (asan)": {
|
||||
"required_build": "package_asan",
|
||||
},
|
||||
"ClickHouse Keeper Jepsen": {
|
||||
"required_build": "binary_release",
|
||||
},
|
||||
"ClickHouse Server Jepsen": {
|
||||
"required_build": "binary_release",
|
||||
},
|
||||
"Performance Comparison": {
|
||||
"required_build": "package_release",
|
||||
"test_grep_exclude_filter": "",
|
||||
},
|
||||
"Performance Comparison Aarch64": {
|
||||
"required_build": "package_aarch64",
|
||||
"test_grep_exclude_filter": "",
|
||||
},
|
||||
"SQLancer (release)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
"SQLancer (debug)": {
|
||||
"required_build": "package_debug",
|
||||
},
|
||||
"Sqllogic test (release)": {
|
||||
"required_build": "package_release",
|
||||
},
|
||||
"Stateless tests (asan)": TestConfig("package_asan"),
|
||||
"Stateless tests (tsan)": TestConfig("package_tsan"),
|
||||
"Stateless tests (msan)": TestConfig("package_msan"),
|
||||
"Stateless tests (ubsan)": TestConfig("package_ubsan"),
|
||||
"Stateless tests (debug)": TestConfig("package_debug"),
|
||||
"Stateless tests (release)": TestConfig("package_release"),
|
||||
"Stateless tests (aarch64)": TestConfig("package_aarch64"),
|
||||
"Stateless tests (release, wide parts enabled)": TestConfig("package_release"),
|
||||
"Stateless tests (release, analyzer)": TestConfig("package_release"),
|
||||
"Stateless tests (release, DatabaseOrdinary)": TestConfig("package_release"),
|
||||
"Stateless tests (release, DatabaseReplicated)": TestConfig("package_release"),
|
||||
"Stateless tests (release, s3 storage)": TestConfig("package_release"),
|
||||
"Stateless tests (debug, s3 storage)": TestConfig("package_debug"),
|
||||
"Stateless tests (tsan, s3 storage)": TestConfig("package_tsan"),
|
||||
"Stress test (asan)": TestConfig("package_asan"),
|
||||
"Stress test (tsan)": TestConfig("package_tsan"),
|
||||
"Stress test (ubsan)": TestConfig("package_ubsan"),
|
||||
"Stress test (msan)": TestConfig("package_msan"),
|
||||
"Stress test (debug)": TestConfig("package_debug"),
|
||||
"Upgrade check (asan)": TestConfig("package_asan"),
|
||||
"Upgrade check (tsan)": TestConfig("package_tsan"),
|
||||
"Upgrade check (msan)": TestConfig("package_msan"),
|
||||
"Upgrade check (debug)": TestConfig("package_debug"),
|
||||
"Integration tests (asan)": TestConfig("package_asan"),
|
||||
"Integration tests (asan, analyzer)": TestConfig("package_asan"),
|
||||
"Integration tests (tsan)": TestConfig("package_tsan"),
|
||||
"Integration tests (release)": TestConfig("package_release"),
|
||||
"Integration tests (msan)": TestConfig("package_msan"),
|
||||
"Integration tests flaky check (asan)": TestConfig("package_asan"),
|
||||
"Compatibility check (amd64)": TestConfig("package_release"),
|
||||
"Compatibility check (aarch64)": TestConfig("package_aarch64"),
|
||||
"Unit tests (release)": TestConfig("binary_release"),
|
||||
"Unit tests (asan)": TestConfig("package_asan"),
|
||||
"Unit tests (msan)": TestConfig("package_msan"),
|
||||
"Unit tests (tsan)": TestConfig("package_tsan"),
|
||||
"Unit tests (ubsan)": TestConfig("package_ubsan"),
|
||||
"AST fuzzer (debug)": TestConfig("package_debug"),
|
||||
"AST fuzzer (asan)": TestConfig("package_asan"),
|
||||
"AST fuzzer (msan)": TestConfig("package_msan"),
|
||||
"AST fuzzer (tsan)": TestConfig("package_tsan"),
|
||||
"AST fuzzer (ubsan)": TestConfig("package_ubsan"),
|
||||
"Stateless tests flaky check (asan)": TestConfig("package_asan"),
|
||||
"ClickHouse Keeper Jepsen": TestConfig("binary_release"),
|
||||
"ClickHouse Server Jepsen": TestConfig("binary_release"),
|
||||
"Performance Comparison": TestConfig("package_release"),
|
||||
"Performance Comparison Aarch64": TestConfig("package_aarch64"),
|
||||
"SQLancer (release)": TestConfig("package_release"),
|
||||
"SQLancer (debug)": TestConfig("package_debug"),
|
||||
"Sqllogic test (release)": TestConfig("package_release"),
|
||||
},
|
||||
} # type: dict
|
||||
)
|
||||
CI_CONFIG.validate()
|
||||
|
||||
|
||||
# checks required by Mergeable Check
|
||||
REQUIRED_CHECKS = [
|
||||
|
||||
@ -51,7 +51,8 @@ class RerunHelper:
|
||||
|
||||
|
||||
def override_status(status: str, check_name: str, invert: bool = False) -> str:
|
||||
if CI_CONFIG["tests_config"].get(check_name, {}).get("force_tests", False):
|
||||
test_config = CI_CONFIG.test_configs.get(check_name)
|
||||
if test_config and test_config.force_tests:
|
||||
return "success"
|
||||
|
||||
if invert:
|
||||
|
||||
@ -9,7 +9,7 @@ import os
|
||||
from pathlib import Path
|
||||
|
||||
from build_download_helper import download_build_with_progress
|
||||
from ci_config import CI_CONFIG, BuildConfig
|
||||
from ci_config import CI_CONFIG
|
||||
from env_helper import RUNNER_TEMP, S3_ARTIFACT_DOWNLOAD_TEMPLATE
|
||||
from git_helper import Git, commit
|
||||
from version_helper import get_version_from_repo, version_arg
|
||||
@ -62,9 +62,9 @@ def main():
|
||||
temp_path.mkdir(parents=True, exist_ok=True)
|
||||
for build in args.build_names:
|
||||
# check if it's in CI_CONFIG
|
||||
config = CI_CONFIG["build_config"][build] # type: BuildConfig
|
||||
if args.rename:
|
||||
path = temp_path / f"clickhouse-{config['static_binary_name']}"
|
||||
config = CI_CONFIG.build_config[build]
|
||||
if args.rename and config.static_binary_name:
|
||||
path = temp_path / f"clickhouse-{config.static_binary_name}"
|
||||
else:
|
||||
path = temp_path / "clickhouse"
|
||||
|
||||
|
||||
@ -71,7 +71,7 @@ if __name__ == "__main__":
|
||||
reports_path = os.getenv("REPORTS_PATH", "./reports")
|
||||
|
||||
check_name = sys.argv[1]
|
||||
required_build = CI_CONFIG["tests_config"][check_name]["required_build"]
|
||||
required_build = CI_CONFIG.test_configs[check_name].required_build
|
||||
|
||||
if not os.path.exists(temp_path):
|
||||
os.makedirs(temp_path)
|
||||
@ -121,15 +121,6 @@ if __name__ == "__main__":
|
||||
)
|
||||
sys.exit(0)
|
||||
|
||||
test_grep_exclude_filter = CI_CONFIG["tests_config"][check_name][
|
||||
"test_grep_exclude_filter"
|
||||
]
|
||||
if test_grep_exclude_filter:
|
||||
docker_env += f" -e CHPC_TEST_GREP_EXCLUDE={test_grep_exclude_filter}"
|
||||
logging.info(
|
||||
"Fill fliter our performance tests by grep -v %s", test_grep_exclude_filter
|
||||
)
|
||||
|
||||
rerun_helper = RerunHelper(commit, check_name_with_group)
|
||||
if rerun_helper.is_already_finished_by_status():
|
||||
logging.info("Check is already finished according to github status, exiting")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user