#!/usr/bin/env python3 import logging from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser from dataclasses import dataclass, field from pathlib import Path from typing import Callable, Dict, Iterable, List, Literal, Optional, Union from ci_utils import WithIter from integration_test_images import IMAGES class Labels(metaclass=WithIter): """ Label names or commit tokens in normalized form """ DO_NOT_TEST_LABEL = "do_not_test" NO_MERGE_COMMIT = "no_merge_commit" NO_CI_CACHE = "no_ci_cache" CI_SET_REDUCED = "ci_set_reduced" CI_SET_ARM = "ci_set_arm" CI_SET_INTEGRATION = "ci_set_integration" class Build(metaclass=WithIter): PACKAGE_RELEASE = "package_release" PACKAGE_AARCH64 = "package_aarch64" PACKAGE_ASAN = "package_asan" PACKAGE_UBSAN = "package_ubsan" PACKAGE_TSAN = "package_tsan" PACKAGE_MSAN = "package_msan" PACKAGE_DEBUG = "package_debug" BINARY_RELEASE = "binary_release" BINARY_TIDY = "binary_tidy" BINARY_DARWIN = "binary_darwin" BINARY_AARCH64 = "binary_aarch64" BINARY_AARCH64_V80COMPAT = "binary_aarch64_v80compat" BINARY_FREEBSD = "binary_freebsd" BINARY_DARWIN_AARCH64 = "binary_darwin_aarch64" BINARY_PPC64LE = "binary_ppc64le" BINARY_AMD64_COMPAT = "binary_amd64_compat" BINARY_AMD64_MUSL = "binary_amd64_musl" BINARY_RISCV64 = "binary_riscv64" BINARY_S390X = "binary_s390x" FUZZERS = "fuzzers" class JobNames(metaclass=WithIter): STYLE_CHECK = "Style check" FAST_TEST = "Fast test" DOCKER_SERVER = "Docker server image" DOCKER_KEEPER = "Docker keeper image" INSTALL_TEST_AMD = "Install packages (amd64)" INSTALL_TEST_ARM = "Install packages (arm64)" STATELESS_TEST_DEBUG = "Stateless tests (debug)" STATELESS_TEST_RELEASE = "Stateless tests (release)" STATELESS_TEST_AARCH64 = "Stateless tests (aarch64)" STATELESS_TEST_ASAN = "Stateless tests (asan)" STATELESS_TEST_TSAN = "Stateless tests (tsan)" STATELESS_TEST_MSAN = "Stateless tests (msan)" STATELESS_TEST_UBSAN = "Stateless tests (ubsan)" STATELESS_TEST_ANALYZER_RELEASE = "Stateless tests (release, analyzer)" STATELESS_TEST_DB_REPL_RELEASE = "Stateless tests (release, DatabaseReplicated)" STATELESS_TEST_S3_RELEASE = "Stateless tests (release, s3 storage)" STATELESS_TEST_S3_DEBUG = "Stateless tests (debug, s3 storage)" STATELESS_TEST_S3_TSAN = "Stateless tests (tsan, s3 storage)" STATELESS_TEST_FLAKY_ASAN = "Stateless tests flaky check (asan)" STATEFUL_TEST_DEBUG = "Stateful tests (debug)" STATEFUL_TEST_RELEASE = "Stateful tests (release)" STATEFUL_TEST_AARCH64 = "Stateful tests (aarch64)" STATEFUL_TEST_ASAN = "Stateful tests (asan)" STATEFUL_TEST_TSAN = "Stateful tests (tsan)" STATEFUL_TEST_MSAN = "Stateful tests (msan)" STATEFUL_TEST_UBSAN = "Stateful tests (ubsan)" STATEFUL_TEST_PARALLEL_REPL_RELEASE = "Stateful tests (release, ParallelReplicas)" STATEFUL_TEST_PARALLEL_REPL_DEBUG = "Stateful tests (debug, ParallelReplicas)" STATEFUL_TEST_PARALLEL_REPL_ASAN = "Stateful tests (asan, ParallelReplicas)" STATEFUL_TEST_PARALLEL_REPL_MSAN = "Stateful tests (msan, ParallelReplicas)" STATEFUL_TEST_PARALLEL_REPL_UBSAN = "Stateful tests (ubsan, ParallelReplicas)" STATEFUL_TEST_PARALLEL_REPL_TSAN = "Stateful tests (tsan, ParallelReplicas)" STRESS_TEST_ASAN = "Stress test (asan)" STRESS_TEST_TSAN = "Stress test (tsan)" STRESS_TEST_UBSAN = "Stress test (ubsan)" STRESS_TEST_MSAN = "Stress test (msan)" STRESS_TEST_DEBUG = "Stress test (debug)" INTEGRATION_TEST = "Integration tests (release)" INTEGRATION_TEST_ASAN = "Integration tests (asan)" INTEGRATION_TEST_ASAN_ANALYZER = "Integration tests (asan, analyzer)" INTEGRATION_TEST_TSAN = "Integration tests (tsan)" INTEGRATION_TEST_ARM = "Integration tests (aarch64)" INTEGRATION_TEST_FLAKY = "Integration tests flaky check (asan)" UPGRADE_TEST_DEBUG = "Upgrade check (debug)" UPGRADE_TEST_ASAN = "Upgrade check (asan)" UPGRADE_TEST_TSAN = "Upgrade check (tsan)" UPGRADE_TEST_MSAN = "Upgrade check (msan)" UNIT_TEST = "Unit tests (release)" UNIT_TEST_ASAN = "Unit tests (asan)" UNIT_TEST_MSAN = "Unit tests (msan)" UNIT_TEST_TSAN = "Unit tests (tsan)" UNIT_TEST_UBSAN = "Unit tests (ubsan)" AST_FUZZER_TEST_DEBUG = "AST fuzzer (debug)" AST_FUZZER_TEST_ASAN = "AST fuzzer (asan)" AST_FUZZER_TEST_MSAN = "AST fuzzer (msan)" AST_FUZZER_TEST_TSAN = "AST fuzzer (tsan)" AST_FUZZER_TEST_UBSAN = "AST fuzzer (ubsan)" JEPSEN_KEEPER = "ClickHouse Keeper Jepsen" JEPSEN_SERVER = "ClickHouse Server Jepsen" PERFORMANCE_TEST_AMD64 = "Performance Comparison" PERFORMANCE_TEST_ARM64 = "Performance Comparison Aarch64" SQL_LOGIC_TEST = "Sqllogic test (release)" SQLANCER = "SQLancer (release)" SQLANCER_DEBUG = "SQLancer (debug)" SQLTEST = "SQLTest" COMPATIBILITY_TEST = "Compatibility check (amd64)" COMPATIBILITY_TEST_ARM = "Compatibility check (aarch64)" CLCIKBENCH_TEST = "ClickBench (amd64)" CLCIKBENCH_TEST_ARM = "ClickBench (aarch64)" LIBFUZZER_TEST = "libFuzzer tests" BUILD_CHECK = "ClickHouse build check" BUILD_CHECK_SPECIAL = "ClickHouse special build check" DOCS_CHECK = "Docs check" BUGFIX_VALIDATE = "tests bugfix validate check" MARK_RELEASE_READY = "Mark Commit Release Ready" # dynamically update JobName with Build jobs for attr_name in dir(Build): if not attr_name.startswith("__") and not callable(getattr(Build, attr_name)): setattr(JobNames, attr_name, getattr(Build, attr_name)) @dataclass class DigestConfig: # all files, dirs to include into digest, glob supported include_paths: List[Union[str, Path]] = field(default_factory=list) # file suffixes to exclude from digest exclude_files: List[str] = field(default_factory=list) # directories to exlude from digest exclude_dirs: List[Union[str, Path]] = field(default_factory=list) # docker names to include into digest docker: List[str] = field(default_factory=list) # git submodules digest git_submodules: bool = False @dataclass class LabelConfig: """ configures different CI scenarious per GH label """ run_jobs: Iterable[str] = frozenset() @dataclass class JobConfig: """ contains config parameters for job execution in CI workflow """ # configures digest calculation for the job digest: DigestConfig = field(default_factory=DigestConfig) # will be triggered for the job if omited in CI workflow yml run_command: str = "" # job timeout timeout: Optional[int] = None # sets number of batches for multi-batch job num_batches: int = 1 # label that enables job in CI, if set digest won't be used run_by_label: str = "" # to run always regardless of the job digest or/and label run_always: bool = False # if the job needs to be run on the release branch, including master (e.g. building packages, docker server). # NOTE: Subsequent runs on the same branch with the similar digest are still considered skippable. required_on_release_branch: bool = False # job is for pr workflow only pr_only: bool = False @dataclass class BuildConfig: name: str compiler: str package_type: Literal["deb", "binary", "fuzzers"] additional_pkgs: bool = False debug_build: bool = False sanitizer: str = "" tidy: bool = False sparse_checkout: bool = False comment: str = "" static_binary_name: str = "" job_config: JobConfig = field( default_factory=lambda: JobConfig( required_on_release_branch=True, digest=DigestConfig( include_paths=[ "./src", "./contrib/*-cmake", "./contrib/consistent-hashing", "./contrib/murmurhash", "./contrib/libfarmhash", "./contrib/pdqsort", "./contrib/cityhash102", "./contrib/sparse-checkout", "./contrib/libmetrohash", "./contrib/update-submodules.sh", "./contrib/CMakeLists.txt", "./CMakeLists.txt", "./PreLoad.cmake", "./cmake", "./base", "./programs", "./packages", "./docker/packager/packager", "./rust", # FIXME: This is a WA to rebuild the CH and recreate the Performance.tar.zst artifact # when there are changes in performance test scripts. # Due to the current design of the perf test we need to rebuild CH when the performance test changes, # otherwise the changes will not be visible in the PerformanceTest job in CI "./tests/performance", ], exclude_files=[".md"], docker=["clickhouse/binary-builder"], git_submodules=True, ), run_command="build_check.py $BUILD_NAME", ) ) def export_env(self, export: bool = False) -> str: def process(field_name: str, field: Union[bool, str]) -> str: if isinstance(field, bool): field = str(field).lower() if export: return f"export BUILD_{field_name.upper()}={repr(field)}" return f"BUILD_{field_name.upper()}={field}" return "\n".join(process(k, v) for k, v in self.__dict__.items()) @dataclass class BuildReportConfig: builds: List[str] job_config: JobConfig = field( default_factory=lambda: JobConfig( digest=DigestConfig( include_paths=[ "./tests/ci/build_report_check.py", "./tests/ci/upload_result_helper.py", ], ), ) ) @dataclass class TestConfig: required_build: str force_tests: bool = False job_config: JobConfig = field(default_factory=JobConfig) BuildConfigs = Dict[str, BuildConfig] BuildsReportConfig = Dict[str, BuildReportConfig] TestConfigs = Dict[str, TestConfig] LabelConfigs = Dict[str, LabelConfig] # common digests configs compatibility_check_digest = DigestConfig( include_paths=["./tests/ci/compatibility_check.py"], docker=["clickhouse/test-old-ubuntu", "clickhouse/test-old-centos"], ) install_check_digest = DigestConfig( include_paths=["./tests/ci/install_check.py"], docker=["clickhouse/install-deb-test", "clickhouse/install-rpm-test"], ) stateless_check_digest = DigestConfig( include_paths=[ "./tests/queries/0_stateless/", "./tests/clickhouse-test", "./tests/config", "./tests/*.txt", ], exclude_files=[".md"], docker=["clickhouse/stateless-test"], ) stateful_check_digest = DigestConfig( include_paths=[ "./tests/queries/1_stateful/", "./tests/clickhouse-test", "./tests/config", "./tests/*.txt", ], exclude_files=[".md"], docker=["clickhouse/stateful-test"], ) stress_check_digest = DigestConfig( include_paths=[ "./tests/queries/0_stateless/", "./tests/queries/1_stateful/", "./tests/clickhouse-test", "./tests/config", "./tests/*.txt", ], exclude_files=[".md"], docker=["clickhouse/stress-test"], ) # FIXME: which tests are upgrade? just python? upgrade_check_digest = DigestConfig( include_paths=["./tests/ci/upgrade_check.py"], exclude_files=[".md"], docker=["clickhouse/upgrade-check"], ) integration_check_digest = DigestConfig( include_paths=["./tests/ci/integration_test_check.py", "./tests/integration"], exclude_files=[".md"], docker=IMAGES.copy(), ) ast_fuzzer_check_digest = DigestConfig( # include_paths=["./tests/ci/ast_fuzzer_check.py"], # exclude_files=[".md"], # docker=["clickhouse/fuzzer"], ) unit_check_digest = DigestConfig( include_paths=["./tests/ci/unit_tests_check.py"], exclude_files=[".md"], docker=["clickhouse/unit-test"], ) perf_check_digest = DigestConfig( include_paths=[ "./tests/ci/performance_comparison_check.py", "./tests/performance/", ], exclude_files=[".md"], docker=["clickhouse/performance-comparison"], ) sqllancer_check_digest = DigestConfig( # include_paths=["./tests/ci/sqlancer_check.py"], # exclude_files=[".md"], # docker=["clickhouse/sqlancer-test"], ) sqllogic_check_digest = DigestConfig( include_paths=["./tests/ci/sqllogic_test.py"], exclude_files=[".md"], docker=["clickhouse/sqllogic-test"], ) sqltest_check_digest = DigestConfig( include_paths=["./tests/ci/sqltest.py"], exclude_files=[".md"], docker=["clickhouse/sqltest"], ) bugfix_validate_check = DigestConfig( include_paths=[ "./tests/queries/0_stateless/", "./tests/ci/integration_test_check.py", "./tests/ci/functional_test_check.py", "./tests/ci/bugfix_validate_check.py", ], exclude_files=[".md"], docker=IMAGES.copy() + [ "clickhouse/stateless-test", ], ) # common test params statless_test_common_params = { "digest": stateless_check_digest, "run_command": 'functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT', "timeout": 10800, } stateful_test_common_params = { "digest": stateful_check_digest, "run_command": 'functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT', "timeout": 3600, } stress_test_common_params = { "digest": stress_check_digest, "run_command": "stress_check.py", } upgrade_test_common_params = { "digest": upgrade_check_digest, "run_command": "upgrade_check.py", } astfuzzer_test_common_params = { "digest": ast_fuzzer_check_digest, "run_command": "ast_fuzzer_check.py", "run_always": True, } integration_test_common_params = { "digest": integration_check_digest, "run_command": 'integration_test_check.py "$CHECK_NAME"', } unit_test_common_params = { "digest": unit_check_digest, "run_command": "unit_tests_check.py", } perf_test_common_params = { "digest": perf_check_digest, "run_command": "performance_comparison_check.py", } sqllancer_test_common_params = { "digest": sqllancer_check_digest, "run_command": "sqlancer_check.py", "run_always": True, } sqllogic_test_params = { "digest": sqllogic_check_digest, "run_command": "sqllogic_test.py", "timeout": 10800, } sql_test_params = { "digest": sqltest_check_digest, "run_command": "sqltest.py", "timeout": 10800, } @dataclass class CiConfig: """ Contains configs for ALL jobs in CI pipeline each config item in the below dicts should be an instance of JobConfig class or inherited from it """ build_config: BuildConfigs builds_report_config: BuildsReportConfig test_configs: TestConfigs other_jobs_configs: TestConfigs label_configs: LabelConfigs def get_label_config(self, label_name: str) -> Optional[LabelConfig]: for label, config in self.label_configs.items(): if self.normalize_string(label_name) == self.normalize_string(label): return config return None def get_job_config(self, check_name: str) -> JobConfig: res = None for config in ( self.build_config, self.builds_report_config, self.test_configs, self.other_jobs_configs, ): if check_name in config: # type: ignore res = config[check_name].job_config # type: ignore break assert ( res is not None ), f"Invalid check_name or CI_CONFIG outdated, config not found for [{check_name}]" return res # type: ignore @staticmethod def normalize_string(input_string: str) -> str: lowercase_string = input_string.lower() normalized_string = ( lowercase_string.replace(" ", "_") .replace("-", "_") .replace("(", "") .replace(")", "") .replace(",", "") ) return normalized_string def get_job_with_parents(self, check_name: str) -> List[str]: res = [] check_name = self.normalize_string(check_name) for config in ( self.build_config, self.builds_report_config, self.test_configs, self.other_jobs_configs, ): for job_name in config: # type: ignore if check_name == self.normalize_string(job_name): res.append(job_name) if isinstance(config[job_name], TestConfig): # type: ignore if config[job_name].required_build: # type: ignore res.append(config[job_name].required_build) # type: ignore elif isinstance(config[job_name], BuildConfig): # type: ignore pass elif isinstance(config[job_name], BuildReportConfig): # type: ignore # add all build jobs as parents for build report check res.extend( [job for job in JobNames if job in self.build_config] ) else: assert ( False ), f"check commit message tags or FIXME: request for job [{check_name}] not yet supported" break assert ( res ), f"Error: Experimantal feature... Invlid request or not supported job [{check_name}]" return res def get_digest_config(self, check_name: str) -> DigestConfig: res = None for config in ( self.other_jobs_configs, self.build_config, self.builds_report_config, self.test_configs, ): if check_name in config: # type: ignore res = config[check_name].job_config.digest # type: ignore assert ( res ), f"Invalid check_name or CI_CONFIG outdated, config not found for [{check_name}]" return res # type: ignore def job_generator(self) -> Iterable[str]: """ traverses all check names in CI pipeline """ for config in ( self.other_jobs_configs, self.build_config, self.builds_report_config, self.test_configs, ): for check_name in config: # type: ignore yield check_name def get_builds_for_report(self, report_name: str) -> List[str]: return self.builds_report_config[report_name].builds @classmethod def is_build_job(cls, job: str) -> bool: return job in Build @classmethod def is_test_job(cls, job: str) -> bool: return ( not cls.is_build_job(job) and not cls.is_build_job(job) and job != JobNames.STYLE_CHECK ) @classmethod def is_docs_job(cls, job: str) -> bool: return job == JobNames.DOCS_CHECK def validate(self) -> None: errors = [] for name, build_config in self.build_config.items(): build_in_reports = False for _, report_config in self.builds_report_config.items(): if name in report_config.builds: build_in_reports = True break # All build configs must belong to build_report_config if not build_in_reports: logging.error("Build name %s does not belong to build reports", name) errors.append(f"Build name {name} does not belong to build reports") # The name should be the same as build_config.name if not build_config.name == name: logging.error( "Build name '%s' does not match the config 'name' value '%s'", name, build_config.name, ) errors.append( f"Build name {name} does not match 'name' value '{build_config.name}'" ) # All build_report_config values should be in build_config.keys() for build_report_name, build_report_config in self.builds_report_config.items(): build_names = build_report_config.builds missed_names = [ name for name in build_names if name not in self.build_config.keys() ] if missed_names: logging.error( "The following names of the build report '%s' " "are missed in build_config: %s", build_report_name, missed_names, ) errors.append( f"The following names of the build report '{build_report_name}' " f"are missed in build_config: {missed_names}", ) # And finally, all of tests' requirements must be in the builds for test_name, test_config in self.test_configs.items(): if test_config.required_build not in self.build_config.keys(): logging.error( "The requierment '%s' for '%s' is not found in builds", test_config, test_name, ) errors.append( f"The requierment '{test_config}' for " f"'{test_name}' is not found in builds" ) if errors: raise KeyError("config contains errors", errors) CI_CONFIG = CiConfig( label_configs={ Labels.DO_NOT_TEST_LABEL: LabelConfig(run_jobs=[JobNames.STYLE_CHECK]), Labels.CI_SET_ARM: LabelConfig( run_jobs=[ # JobNames.STYLE_CHECK, Build.PACKAGE_AARCH64, JobNames.INTEGRATION_TEST_ARM, ] ), Labels.CI_SET_INTEGRATION: LabelConfig( run_jobs=[ JobNames.STYLE_CHECK, Build.PACKAGE_ASAN, Build.PACKAGE_RELEASE, Build.PACKAGE_TSAN, Build.PACKAGE_AARCH64, JobNames.INTEGRATION_TEST_ASAN, JobNames.INTEGRATION_TEST_ARM, JobNames.INTEGRATION_TEST, JobNames.INTEGRATION_TEST_ASAN_ANALYZER, JobNames.INTEGRATION_TEST_TSAN, JobNames.INTEGRATION_TEST_FLAKY, ] ), Labels.CI_SET_REDUCED: LabelConfig( run_jobs=[ job for job in JobNames if not any( [ nogo in job for nogo in ( "asan", "tsan", "msan", "ubsan", # skip build report jobs as not all builds will be done "build check", ) ] ) ] ), }, build_config={ Build.PACKAGE_RELEASE: BuildConfig( name=Build.PACKAGE_RELEASE, compiler="clang-17", package_type="deb", static_binary_name="amd64", additional_pkgs=True, ), Build.PACKAGE_AARCH64: BuildConfig( name=Build.PACKAGE_AARCH64, compiler="clang-17-aarch64", package_type="deb", static_binary_name="aarch64", additional_pkgs=True, ), Build.PACKAGE_ASAN: BuildConfig( name=Build.PACKAGE_ASAN, compiler="clang-17", sanitizer="address", package_type="deb", ), Build.PACKAGE_UBSAN: BuildConfig( name=Build.PACKAGE_UBSAN, compiler="clang-17", sanitizer="undefined", package_type="deb", ), Build.PACKAGE_TSAN: BuildConfig( name=Build.PACKAGE_TSAN, compiler="clang-17", sanitizer="thread", package_type="deb", ), Build.PACKAGE_MSAN: BuildConfig( name=Build.PACKAGE_MSAN, compiler="clang-17", sanitizer="memory", package_type="deb", ), Build.PACKAGE_DEBUG: BuildConfig( name=Build.PACKAGE_DEBUG, compiler="clang-17", debug_build=True, package_type="deb", sparse_checkout=True, # Check that it works with at least one build, see also update-submodules.sh ), Build.BINARY_RELEASE: BuildConfig( name=Build.BINARY_RELEASE, compiler="clang-17", package_type="binary", ), Build.BINARY_TIDY: BuildConfig( name=Build.BINARY_TIDY, compiler="clang-17", debug_build=True, package_type="binary", static_binary_name="debug-amd64", tidy=True, comment="clang-tidy is used for static analysis", ), Build.BINARY_DARWIN: BuildConfig( name=Build.BINARY_DARWIN, compiler="clang-17-darwin", package_type="binary", static_binary_name="macos", sparse_checkout=True, # Check that it works with at least one build, see also update-submodules.sh ), Build.BINARY_AARCH64: BuildConfig( name=Build.BINARY_AARCH64, compiler="clang-17-aarch64", package_type="binary", ), Build.BINARY_AARCH64_V80COMPAT: BuildConfig( name=Build.BINARY_AARCH64_V80COMPAT, compiler="clang-17-aarch64-v80compat", package_type="binary", static_binary_name="aarch64v80compat", comment="For ARMv8.1 and older", ), Build.BINARY_FREEBSD: BuildConfig( name=Build.BINARY_FREEBSD, compiler="clang-17-freebsd", package_type="binary", static_binary_name="freebsd", ), Build.BINARY_DARWIN_AARCH64: BuildConfig( name=Build.BINARY_DARWIN_AARCH64, compiler="clang-17-darwin-aarch64", package_type="binary", static_binary_name="macos-aarch64", ), Build.BINARY_PPC64LE: BuildConfig( name=Build.BINARY_PPC64LE, compiler="clang-17-ppc64le", package_type="binary", static_binary_name="powerpc64le", ), Build.BINARY_AMD64_COMPAT: BuildConfig( name=Build.BINARY_AMD64_COMPAT, compiler="clang-17-amd64-compat", package_type="binary", static_binary_name="amd64compat", comment="SSE2-only build", ), Build.BINARY_AMD64_MUSL: BuildConfig( name=Build.BINARY_AMD64_MUSL, compiler="clang-17-amd64-musl", package_type="binary", static_binary_name="amd64musl", comment="Build with Musl", ), Build.BINARY_RISCV64: BuildConfig( name=Build.BINARY_RISCV64, compiler="clang-17-riscv64", package_type="binary", static_binary_name="riscv64", ), Build.BINARY_S390X: BuildConfig( name=Build.BINARY_S390X, compiler="clang-17-s390x", package_type="binary", static_binary_name="s390x", ), Build.FUZZERS: BuildConfig( name=Build.FUZZERS, compiler="clang-17", package_type="fuzzers", ), }, builds_report_config={ JobNames.BUILD_CHECK: BuildReportConfig( builds=[ Build.PACKAGE_RELEASE, Build.PACKAGE_AARCH64, Build.PACKAGE_ASAN, Build.PACKAGE_UBSAN, Build.PACKAGE_TSAN, Build.PACKAGE_MSAN, Build.PACKAGE_DEBUG, Build.BINARY_RELEASE, Build.FUZZERS, ] ), JobNames.BUILD_CHECK_SPECIAL: BuildReportConfig( builds=[ Build.BINARY_TIDY, Build.BINARY_DARWIN, Build.BINARY_AARCH64, Build.BINARY_AARCH64_V80COMPAT, Build.BINARY_FREEBSD, Build.BINARY_DARWIN_AARCH64, Build.BINARY_PPC64LE, Build.BINARY_RISCV64, Build.BINARY_S390X, Build.BINARY_AMD64_COMPAT, Build.BINARY_AMD64_MUSL, ] ), }, other_jobs_configs={ JobNames.MARK_RELEASE_READY: TestConfig( "", job_config=JobConfig(required_on_release_branch=True) ), JobNames.DOCKER_SERVER: TestConfig( "", job_config=JobConfig( required_on_release_branch=True, digest=DigestConfig( include_paths=[ "tests/ci/docker_server.py", "./docker/server", ] ), ), ), JobNames.DOCKER_KEEPER: TestConfig( "", job_config=JobConfig( digest=DigestConfig( include_paths=[ "tests/ci/docker_server.py", "./docker/keeper", ] ), ), ), JobNames.DOCS_CHECK: TestConfig( "", job_config=JobConfig( digest=DigestConfig( include_paths=["**/*.md", "./docs", "tests/ci/docs_check.py"], docker=["clickhouse/docs-builder"], ), ), ), JobNames.FAST_TEST: TestConfig( "", job_config=JobConfig( pr_only=True, digest=DigestConfig( include_paths=["./tests/queries/0_stateless/"], exclude_files=[".md"], docker=["clickhouse/fasttest"], ), ), ), JobNames.STYLE_CHECK: TestConfig( "", job_config=JobConfig( run_always=True, ), ), JobNames.BUGFIX_VALIDATE: TestConfig( "", # we run this check by label - no digest required job_config=JobConfig(run_by_label="pr-bugfix"), ), }, test_configs={ JobNames.INSTALL_TEST_AMD: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig(digest=install_check_digest) ), JobNames.INSTALL_TEST_ARM: TestConfig( Build.PACKAGE_AARCH64, job_config=JobConfig(digest=install_check_digest) ), JobNames.STATEFUL_TEST_ASAN: TestConfig( Build.PACKAGE_ASAN, job_config=JobConfig(**stateful_test_common_params) # type: ignore ), JobNames.STATEFUL_TEST_TSAN: TestConfig( Build.PACKAGE_TSAN, job_config=JobConfig(**stateful_test_common_params) # type: ignore ), JobNames.STATEFUL_TEST_MSAN: TestConfig( Build.PACKAGE_MSAN, job_config=JobConfig(**stateful_test_common_params) # type: ignore ), JobNames.STATEFUL_TEST_UBSAN: TestConfig( Build.PACKAGE_UBSAN, job_config=JobConfig(**stateful_test_common_params) # type: ignore ), JobNames.STATEFUL_TEST_DEBUG: TestConfig( Build.PACKAGE_DEBUG, job_config=JobConfig(**stateful_test_common_params) # type: ignore ), JobNames.STATEFUL_TEST_RELEASE: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig(**stateful_test_common_params) # type: ignore ), JobNames.STATEFUL_TEST_AARCH64: TestConfig( Build.PACKAGE_AARCH64, job_config=JobConfig(**stateful_test_common_params) # type: ignore ), # FIXME: delete? # "Stateful tests (release, DatabaseOrdinary)": TestConfig( # Build.PACKAGE_RELEASE, job_config=JobConfig(**stateful_test_common_params) # type: ignore # ), # "Stateful tests (release, DatabaseReplicated)": TestConfig( # Build.PACKAGE_RELEASE, job_config=JobConfig(**stateful_test_common_params) # type: ignore # ), # Stateful tests for parallel replicas JobNames.STATEFUL_TEST_PARALLEL_REPL_RELEASE: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig(**stateful_test_common_params) # type: ignore ), JobNames.STATEFUL_TEST_PARALLEL_REPL_DEBUG: TestConfig( Build.PACKAGE_DEBUG, job_config=JobConfig(**stateful_test_common_params) # type: ignore ), JobNames.STATEFUL_TEST_PARALLEL_REPL_ASAN: TestConfig( Build.PACKAGE_ASAN, job_config=JobConfig(**stateful_test_common_params) # type: ignore ), JobNames.STATEFUL_TEST_PARALLEL_REPL_MSAN: TestConfig( Build.PACKAGE_MSAN, job_config=JobConfig(**stateful_test_common_params) # type: ignore ), JobNames.STATEFUL_TEST_PARALLEL_REPL_UBSAN: TestConfig( Build.PACKAGE_UBSAN, job_config=JobConfig(**stateful_test_common_params) # type: ignore ), JobNames.STATEFUL_TEST_PARALLEL_REPL_TSAN: TestConfig( Build.PACKAGE_TSAN, job_config=JobConfig(**stateful_test_common_params) # type: ignore ), # End stateful tests for parallel replicas JobNames.STATELESS_TEST_ASAN: TestConfig( Build.PACKAGE_ASAN, job_config=JobConfig(num_batches=4, **statless_test_common_params), # type: ignore ), JobNames.STATELESS_TEST_TSAN: TestConfig( Build.PACKAGE_TSAN, job_config=JobConfig(num_batches=5, **statless_test_common_params), # type: ignore ), JobNames.STATELESS_TEST_MSAN: TestConfig( Build.PACKAGE_MSAN, job_config=JobConfig(num_batches=6, **statless_test_common_params), # type: ignore ), JobNames.STATELESS_TEST_UBSAN: TestConfig( Build.PACKAGE_UBSAN, job_config=JobConfig(num_batches=2, **statless_test_common_params), # type: ignore ), JobNames.STATELESS_TEST_DEBUG: TestConfig( Build.PACKAGE_DEBUG, job_config=JobConfig(num_batches=5, **statless_test_common_params), # type: ignore ), JobNames.STATELESS_TEST_RELEASE: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig(**statless_test_common_params) # type: ignore ), JobNames.STATELESS_TEST_AARCH64: TestConfig( Build.PACKAGE_AARCH64, job_config=JobConfig(**statless_test_common_params) # type: ignore ), JobNames.STATELESS_TEST_ANALYZER_RELEASE: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig(**statless_test_common_params) # type: ignore ), JobNames.STATELESS_TEST_DB_REPL_RELEASE: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig(num_batches=4, **statless_test_common_params), # type: ignore ), JobNames.STATELESS_TEST_S3_RELEASE: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig(num_batches=2, **statless_test_common_params), # type: ignore ), JobNames.STATELESS_TEST_S3_DEBUG: TestConfig( Build.PACKAGE_DEBUG, job_config=JobConfig(num_batches=6, **statless_test_common_params), # type: ignore ), JobNames.STATELESS_TEST_S3_TSAN: TestConfig( Build.PACKAGE_TSAN, job_config=JobConfig(num_batches=5, **statless_test_common_params), # type: ignore ), JobNames.STRESS_TEST_ASAN: TestConfig( Build.PACKAGE_ASAN, job_config=JobConfig(**stress_test_common_params) # type: ignore ), JobNames.STRESS_TEST_TSAN: TestConfig( Build.PACKAGE_TSAN, job_config=JobConfig(**stress_test_common_params) # type: ignore ), JobNames.STRESS_TEST_UBSAN: TestConfig( Build.PACKAGE_UBSAN, job_config=JobConfig(**stress_test_common_params) # type: ignore ), JobNames.STRESS_TEST_MSAN: TestConfig( Build.PACKAGE_MSAN, job_config=JobConfig(**stress_test_common_params) # type: ignore ), JobNames.STRESS_TEST_DEBUG: TestConfig( Build.PACKAGE_DEBUG, job_config=JobConfig(**stress_test_common_params) # type: ignore ), JobNames.UPGRADE_TEST_ASAN: TestConfig( Build.PACKAGE_ASAN, job_config=JobConfig(**upgrade_test_common_params) # type: ignore ), JobNames.UPGRADE_TEST_TSAN: TestConfig( Build.PACKAGE_TSAN, job_config=JobConfig(**upgrade_test_common_params) # type: ignore ), JobNames.UPGRADE_TEST_MSAN: TestConfig( Build.PACKAGE_MSAN, job_config=JobConfig(**upgrade_test_common_params) # type: ignore ), JobNames.UPGRADE_TEST_DEBUG: TestConfig( Build.PACKAGE_DEBUG, job_config=JobConfig(**upgrade_test_common_params) # type: ignore ), JobNames.INTEGRATION_TEST_ASAN: TestConfig( Build.PACKAGE_ASAN, job_config=JobConfig(num_batches=4, **integration_test_common_params), # type: ignore ), JobNames.INTEGRATION_TEST_ASAN_ANALYZER: TestConfig( Build.PACKAGE_ASAN, job_config=JobConfig(num_batches=6, **integration_test_common_params), # type: ignore ), JobNames.INTEGRATION_TEST_TSAN: TestConfig( Build.PACKAGE_TSAN, job_config=JobConfig(num_batches=6, **integration_test_common_params), # type: ignore ), JobNames.INTEGRATION_TEST_ARM: TestConfig( Build.PACKAGE_AARCH64, # add [run_by_label="test arm"] to not run in regular pr workflow by default job_config=JobConfig(num_batches=6, **integration_test_common_params, run_by_label="test arm"), # type: ignore ), # FIXME: currently no wf has this job. Try to enable # "Integration tests (msan)": TestConfig(Build.PACKAGE_MSAN, job_config=JobConfig(num_batches=6, **integration_test_common_params) # type: ignore # ), JobNames.INTEGRATION_TEST: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig(num_batches=4, **integration_test_common_params), # type: ignore ), JobNames.INTEGRATION_TEST_FLAKY: TestConfig( Build.PACKAGE_ASAN, job_config=JobConfig(**integration_test_common_params) # type: ignore ), JobNames.COMPATIBILITY_TEST: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig( required_on_release_branch=True, digest=compatibility_check_digest ), ), JobNames.COMPATIBILITY_TEST_ARM: TestConfig( Build.PACKAGE_AARCH64, job_config=JobConfig( required_on_release_branch=True, digest=compatibility_check_digest ), ), JobNames.UNIT_TEST: TestConfig( Build.BINARY_RELEASE, job_config=JobConfig(**unit_test_common_params) # type: ignore ), JobNames.UNIT_TEST_ASAN: TestConfig( Build.PACKAGE_ASAN, job_config=JobConfig(**unit_test_common_params) # type: ignore ), JobNames.UNIT_TEST_MSAN: TestConfig( Build.PACKAGE_MSAN, job_config=JobConfig(**unit_test_common_params) # type: ignore ), JobNames.UNIT_TEST_TSAN: TestConfig( Build.PACKAGE_TSAN, job_config=JobConfig(**unit_test_common_params) # type: ignore ), JobNames.UNIT_TEST_UBSAN: TestConfig( Build.PACKAGE_UBSAN, job_config=JobConfig(**unit_test_common_params) # type: ignore ), JobNames.AST_FUZZER_TEST_DEBUG: TestConfig( Build.PACKAGE_DEBUG, job_config=JobConfig(**astfuzzer_test_common_params) # type: ignore ), JobNames.AST_FUZZER_TEST_ASAN: TestConfig( Build.PACKAGE_ASAN, job_config=JobConfig(**astfuzzer_test_common_params) # type: ignore ), JobNames.AST_FUZZER_TEST_MSAN: TestConfig( Build.PACKAGE_MSAN, job_config=JobConfig(**astfuzzer_test_common_params) # type: ignore ), JobNames.AST_FUZZER_TEST_TSAN: TestConfig( Build.PACKAGE_TSAN, job_config=JobConfig(**astfuzzer_test_common_params) # type: ignore ), JobNames.AST_FUZZER_TEST_UBSAN: TestConfig( Build.PACKAGE_UBSAN, job_config=JobConfig(**astfuzzer_test_common_params) # type: ignore ), JobNames.STATELESS_TEST_FLAKY_ASAN: TestConfig( # replace to non-default Build.PACKAGE_ASAN, job_config=JobConfig(**{**statless_test_common_params, "timeout": 3600}), # type: ignore ), JobNames.JEPSEN_KEEPER: TestConfig( Build.BINARY_RELEASE, job_config=JobConfig( run_by_label="jepsen-test", run_command="jepsen_check.py keeper" ), ), JobNames.JEPSEN_SERVER: TestConfig( Build.BINARY_RELEASE, job_config=JobConfig( run_by_label="jepsen-test", run_command="jepsen_check.py server" ), ), JobNames.PERFORMANCE_TEST_AMD64: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig(num_batches=4, **perf_test_common_params), # type: ignore ), JobNames.PERFORMANCE_TEST_ARM64: TestConfig( Build.PACKAGE_AARCH64, job_config=JobConfig(num_batches=4, run_by_label="pr-performance", **perf_test_common_params), # type: ignore ), JobNames.SQLANCER: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig(**sqllancer_test_common_params) # type: ignore ), JobNames.SQLANCER_DEBUG: TestConfig( Build.PACKAGE_DEBUG, job_config=JobConfig(**sqllancer_test_common_params) # type: ignore ), JobNames.SQL_LOGIC_TEST: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig(**sqllogic_test_params) # type: ignore ), JobNames.SQLTEST: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig(**sql_test_params) # type: ignore ), JobNames.CLCIKBENCH_TEST: TestConfig(Build.PACKAGE_RELEASE), JobNames.CLCIKBENCH_TEST_ARM: TestConfig(Build.PACKAGE_AARCH64), JobNames.LIBFUZZER_TEST: TestConfig(Build.FUZZERS), # type: ignore }, ) CI_CONFIG.validate() # checks required by Mergeable Check REQUIRED_CHECKS = [ "PR Check", JobNames.BUILD_CHECK, JobNames.BUILD_CHECK_SPECIAL, JobNames.DOCS_CHECK, JobNames.FAST_TEST, JobNames.STATEFUL_TEST_RELEASE, JobNames.STATELESS_TEST_RELEASE, JobNames.STYLE_CHECK, JobNames.UNIT_TEST_ASAN, JobNames.UNIT_TEST_MSAN, JobNames.UNIT_TEST, JobNames.UNIT_TEST_TSAN, JobNames.UNIT_TEST_UBSAN, ] @dataclass class CheckDescription: name: str description: str # the check descriptions, will be put into the status table match_func: Callable[[str], bool] # the function to check vs the commit status def __hash__(self) -> int: return hash(self.name + self.description) CHECK_DESCRIPTIONS = [ CheckDescription( "AST fuzzer", "Runs randomly generated queries to catch program errors. " "The build type is optionally given in parenthesis. " "If it fails, ask a maintainer for help", lambda x: x.startswith("AST fuzzer"), ), CheckDescription( "Bugfix validate check", "Checks that either a new test (functional or integration) or there " "some changed tests that fail with the binary built on master branch", lambda x: x == "Bugfix validate check", ), CheckDescription( "CI running", "A meta-check that indicates the running CI. Normally, it's in success or " "pending state. The failed status indicates some problems with the PR", lambda x: x == "CI running", ), CheckDescription( "ClickHouse build check", "Builds ClickHouse in various configurations for use in further steps. " "You have to fix the builds that fail. Build logs often has enough " "information to fix the error, but you might have to reproduce the failure " "locally. The cmake options can be found in the build log, grepping for " 'cmake. Use these options and follow the general build process', lambda x: x.startswith("ClickHouse") and x.endswith("build check"), ), CheckDescription( "Compatibility check", "Checks that clickhouse binary runs on distributions with old libc " "versions. If it fails, ask a maintainer for help", lambda x: x.startswith("Compatibility check"), ), CheckDescription( JobNames.DOCKER_SERVER, "The check to build and optionally push the mentioned image to docker hub", lambda x: x.startswith("Docker server"), ), CheckDescription( JobNames.DOCKER_KEEPER, "The check to build and optionally push the mentioned image to docker hub", lambda x: x.startswith("Docker keeper"), ), CheckDescription( JobNames.DOCS_CHECK, "Builds and tests the documentation", lambda x: x == JobNames.DOCS_CHECK, ), CheckDescription( JobNames.FAST_TEST, "Normally this is the first check that is ran for a PR. It builds ClickHouse " 'and runs most of stateless functional tests, ' "omitting some. If it fails, further checks are not started until it is fixed. " "Look at the report to see which tests fail, then reproduce the failure " 'locally as described here', lambda x: x == JobNames.FAST_TEST, ), CheckDescription( "Flaky tests", "Checks if new added or modified tests are flaky by running them repeatedly, " "in parallel, with more randomization. Functional tests are run 100 times " "with address sanitizer, and additional randomization of thread scheduling. " "Integrational tests are run up to 10 times. If at least once a new test has " "failed, or was too long, this check will be red. We don't allow flaky tests, " 'read the doc', lambda x: "tests flaky check" in x, ), CheckDescription( "Install packages", "Checks that the built packages are installable in a clear environment", lambda x: x.startswith("Install packages ("), ), CheckDescription( "Integration tests", "The integration tests report. In parenthesis the package type is given, " "and in square brackets are the optional part/total tests", lambda x: x.startswith("Integration tests ("), ), CheckDescription( "Mergeable Check", "Checks if all other necessary checks are successful", lambda x: x == "Mergeable Check", ), CheckDescription( "Performance Comparison", "Measure changes in query performance. The performance test report is " 'described in detail here. ' "In square brackets are the optional part/total tests", lambda x: x.startswith("Performance Comparison"), ), CheckDescription( "Push to Dockerhub", "The check for building and pushing the CI related docker images to docker hub", lambda x: x.startswith("Push") and "to Dockerhub" in x, ), CheckDescription( "Sqllogic", "Run clickhouse on the " 'sqllogic ' "test set against sqlite and checks that all statements are passed", lambda x: x.startswith("Sqllogic test"), ), CheckDescription( "SQLancer", "Fuzzing tests that detect logical bugs with " 'SQLancer tool', lambda x: x.startswith("SQLancer"), ), CheckDescription( "Stateful tests", "Runs stateful functional tests for ClickHouse binaries built in various " "configurations -- release, debug, with sanitizers, etc", lambda x: x.startswith("Stateful tests ("), ), CheckDescription( "Stateless tests", "Runs stateless functional tests for ClickHouse binaries built in various " "configurations -- release, debug, with sanitizers, etc", lambda x: x.startswith("Stateless tests ("), ), CheckDescription( "Stress test", "Runs stateless functional tests concurrently from several clients to detect " "concurrency-related errors", lambda x: x.startswith("Stress test ("), ), CheckDescription( JobNames.STYLE_CHECK, "Runs a set of checks to keep the code style clean. If some of tests failed, " "see the related log from the report", lambda x: x == JobNames.STYLE_CHECK, ), CheckDescription( "Unit tests", "Runs the unit tests for different release types", lambda x: x.startswith("Unit tests ("), ), CheckDescription( "Upgrade check", "Runs stress tests on server version from last release and then tries to " "upgrade it to the version from the PR. It checks if the new server can " "successfully startup without any errors, crashes or sanitizer asserts", lambda x: x.startswith("Upgrade check ("), ), CheckDescription( "ClickBench", "Runs [ClickBench](https://github.com/ClickHouse/ClickBench/) with instant-attach table", lambda x: x.startswith("ClickBench"), ), CheckDescription( "Falback for unknown", "There's no description for the check yet, please add it to " "tests/ci/ci_config.py:CHECK_DESCRIPTIONS", lambda x: True, ), ] def main() -> None: parser = ArgumentParser( formatter_class=ArgumentDefaultsHelpFormatter, description="The script provides build config for GITHUB_ENV or shell export", ) parser.add_argument("--build-name", help="the build config to export") parser.add_argument( "--export", action="store_true", help="if set, the ENV parameters are provided for shell export", ) args = parser.parse_args() build_config = CI_CONFIG.build_config.get(args.build_name) if build_config: print(build_config.export_env(args.export)) if __name__ == "__main__": main()