#!/usr/bin/env python3 from dataclasses import dataclass from typing import Callable, Dict, TypeVar ConfValue = TypeVar("ConfValue", str, bool) BuildConfig = Dict[str, ConfValue] CI_CONFIG = { "build_config": { "package_release": { "compiler": "clang-16", "build_type": "", "sanitizer": "", "package_type": "deb", "static_binary_name": "amd64", "additional_pkgs": True, "tidy": "disable", "with_coverage": False, }, "coverity": { "compiler": "clang-16", "build_type": "", "sanitizer": "", "package_type": "coverity", "tidy": "disable", "with_coverage": False, "official": False, }, "package_aarch64": { "compiler": "clang-16-aarch64", "build_type": "", "sanitizer": "", "package_type": "deb", "static_binary_name": "aarch64", "additional_pkgs": True, "tidy": "disable", "with_coverage": False, }, "package_asan": { "compiler": "clang-16", "build_type": "", "sanitizer": "address", "package_type": "deb", "tidy": "disable", "with_coverage": False, }, "package_ubsan": { "compiler": "clang-16", "build_type": "", "sanitizer": "undefined", "package_type": "deb", "tidy": "disable", "with_coverage": False, }, "package_tsan": { "compiler": "clang-16", "build_type": "", "sanitizer": "thread", "package_type": "deb", "tidy": "disable", "with_coverage": False, }, "package_msan": { "compiler": "clang-16", "build_type": "", "sanitizer": "memory", "package_type": "deb", "tidy": "disable", "with_coverage": False, }, "package_debug": { "compiler": "clang-16", "build_type": "debug", "sanitizer": "", "package_type": "deb", "tidy": "disable", "with_coverage": False, }, "binary_release": { "compiler": "clang-16", "build_type": "", "sanitizer": "", "package_type": "binary", "tidy": "disable", "with_coverage": False, }, "binary_tidy": { "compiler": "clang-16", "build_type": "debug", "sanitizer": "", "package_type": "binary", "static_binary_name": "debug-amd64", "tidy": "enable", "with_coverage": False, }, "binary_darwin": { "compiler": "clang-16-darwin", "build_type": "", "sanitizer": "", "package_type": "binary", "static_binary_name": "macos", "tidy": "disable", "with_coverage": False, }, "binary_aarch64": { "compiler": "clang-16-aarch64", "build_type": "", "sanitizer": "", "package_type": "binary", "tidy": "disable", "with_coverage": False, }, "binary_aarch64_v80compat": { "compiler": "clang-16-aarch64-v80compat", "build_type": "", "sanitizer": "", "package_type": "binary", "static_binary_name": "aarch64v80compat", "tidy": "disable", "with_coverage": False, }, "binary_freebsd": { "compiler": "clang-16-freebsd", "build_type": "", "sanitizer": "", "package_type": "binary", "static_binary_name": "freebsd", "tidy": "disable", "with_coverage": False, }, "binary_darwin_aarch64": { "compiler": "clang-16-darwin-aarch64", "build_type": "", "sanitizer": "", "package_type": "binary", "static_binary_name": "macos-aarch64", "tidy": "disable", "with_coverage": False, }, "binary_ppc64le": { "compiler": "clang-16-ppc64le", "build_type": "", "sanitizer": "", "package_type": "binary", "static_binary_name": "powerpc64le", "tidy": "disable", "with_coverage": False, }, "binary_amd64_compat": { "compiler": "clang-16-amd64-compat", "build_type": "", "sanitizer": "", "package_type": "binary", "static_binary_name": "amd64compat", "tidy": "disable", "with_coverage": False, }, }, "builds_report_config": { "ClickHouse build check": [ "package_release", "coverity", "package_aarch64", "package_asan", "package_ubsan", "package_tsan", "package_msan", "package_debug", "binary_release", ], "ClickHouse special build check": [ "binary_tidy", "binary_darwin", "binary_aarch64", "binary_aarch64_v80compat", "binary_freebsd", "binary_darwin_aarch64", "binary_ppc64le", "binary_amd64_compat", ], }, "tests_config": { # required_build - build name for artifacts # force_tests - force success status for tests "Install packages (amd64)": { "required_build": "package_release", }, "Install packages (arm64)": { "required_build": "package_aarch64", }, "Stateful tests (asan)": { "required_build": "package_asan", }, "Stateful tests (tsan)": { "required_build": "package_tsan", }, "Stateful tests (msan)": { "required_build": "package_msan", }, "Stateful tests (ubsan)": { "required_build": "package_ubsan", }, "Stateful tests (debug)": { "required_build": "package_debug", }, "Stateful tests (release)": { "required_build": "package_release", }, "Stateful tests (aarch64)": { "required_build": "package_aarch64", }, "Stateful tests (release, DatabaseOrdinary)": { "required_build": "package_release", }, "Stateful tests (release, DatabaseReplicated)": { "required_build": "package_release", }, # Stateful tests for parallel replicas "Stateful tests (release, ParallelReplicas)": { "required_build": "package_release", }, "Stateful tests (debug, ParallelReplicas)": { "required_build": "package_debug", }, "Stateful tests (asan, ParallelReplicas)": { "required_build": "package_asan", }, "Stateful tests (msan, ParallelReplicas)": { "required_build": "package_msan", }, "Stateful tests (ubsan, ParallelReplicas)": { "required_build": "package_ubsan", }, "Stateful tests (tsan, ParallelReplicas)": { "required_build": "package_tsan", }, # End stateful tests for parallel replicas "Stateless tests (asan)": { "required_build": "package_asan", }, "Stateless tests (tsan)": { "required_build": "package_tsan", }, "Stateless tests (msan)": { "required_build": "package_msan", }, "Stateless tests (ubsan)": { "required_build": "package_ubsan", }, "Stateless tests (debug)": { "required_build": "package_debug", }, "Stateless tests (release)": { "required_build": "package_release", }, "Stateless tests (aarch64)": { "required_build": "package_aarch64", }, "Stateless tests (release, wide parts enabled)": { "required_build": "package_release", }, "Stateless tests (release, analyzer)": { "required_build": "package_release", }, "Stateless tests (release, DatabaseOrdinary)": { "required_build": "package_release", }, "Stateless tests (release, DatabaseReplicated)": { "required_build": "package_release", }, "Stateless tests (release, s3 storage)": { "required_build": "package_release", }, "Stateless tests (debug, s3 storage)": { "required_build": "package_debug", }, "Stateless tests (tsan, s3 storage)": { "required_build": "package_tsan", }, "Stress test (asan)": { "required_build": "package_asan", }, "Stress test (tsan)": { "required_build": "package_tsan", }, "Stress test (ubsan)": { "required_build": "package_ubsan", }, "Stress test (msan)": { "required_build": "package_msan", }, "Stress test (debug)": { "required_build": "package_debug", }, "Upgrade check (asan)": { "required_build": "package_asan", }, "Upgrade check (tsan)": { "required_build": "package_tsan", }, "Upgrade check (msan)": { "required_build": "package_msan", }, "Upgrade check (debug)": { "required_build": "package_debug", }, "Integration tests (asan)": { "required_build": "package_asan", }, "Integration tests (tsan)": { "required_build": "package_tsan", }, "Integration tests (release)": { "required_build": "package_release", }, "Integration tests (msan)": { "required_build": "package_msan", }, "Integration tests flaky check (asan)": { "required_build": "package_asan", }, "Compatibility check (amd64)": { "required_build": "package_release", }, "Compatibility check (aarch64)": { "required_build": "package_aarch64", }, "Unit tests (release-clang)": { "required_build": "binary_release", }, "Unit tests (asan)": { "required_build": "package_asan", }, "Unit tests (msan)": { "required_build": "package_msan", }, "Unit tests (tsan)": { "required_build": "package_tsan", }, "Unit tests (ubsan)": { "required_build": "package_ubsan", }, "AST fuzzer (debug)": { "required_build": "package_debug", }, "AST fuzzer (asan)": { "required_build": "package_asan", }, "AST fuzzer (msan)": { "required_build": "package_msan", }, "AST fuzzer (tsan)": { "required_build": "package_tsan", }, "AST fuzzer (ubsan)": { "required_build": "package_ubsan", }, "Stateless tests flaky check (asan)": { "required_build": "package_asan", }, "ClickHouse Keeper Jepsen": { "required_build": "binary_release", }, "ClickHouse Server Jepsen": { "required_build": "binary_release", }, "Performance Comparison": { "required_build": "package_release", "test_grep_exclude_filter": "", }, "Performance Comparison Aarch64": { "required_build": "package_aarch64", "test_grep_exclude_filter": "", }, "SQLancer (release)": { "required_build": "package_release", }, "SQLancer (debug)": { "required_build": "package_debug", }, "Sqllogic test (release)": { "required_build": "package_release", }, }, } # type: dict # checks required by Mergeable Check REQUIRED_CHECKS = [ "ClickHouse build check", "ClickHouse special build check", "Docs Check", "Fast test", "Stateful tests (release)", "Stateless tests (release)", "Style Check", "Unit tests (asan)", "Unit tests (msan)", "Unit tests (release-clang)", "Unit tests (tsan)", "Unit tests (ubsan)", ] @dataclass class CheckDescription: name: str description: str # the check descriptions, will be put into the status table match_func: Callable[[str], bool] # the function to check vs the commit status def __hash__(self) -> int: return hash(self.name + self.description) CHECK_DESCRIPTIONS = [ CheckDescription( "AST fuzzer", "Runs randomly generated queries to catch program errors. " "The build type is optionally given in parenthesis. " "If it fails, ask a maintainer for help", lambda x: x.startswith("AST fuzzer"), ), CheckDescription( "Bugfix validate check", "Checks that either a new test (functional or integration) or there " "some changed tests that fail with the binary built on master branch", lambda x: x == "Bugfix validate check", ), CheckDescription( "CI running", "A meta-check that indicates the running CI. Normally, it's in success or " "pending state. The failed status indicates some problems with the PR", lambda x: x == "CI running", ), CheckDescription( "ClickHouse build check", "Builds ClickHouse in various configurations for use in further steps. " "You have to fix the builds that fail. Build logs often has enough " "information to fix the error, but you might have to reproduce the failure " "locally. The cmake options can be found in the build log, grepping for " 'cmake. Use these options and follow the general build process', lambda x: x.startswith("ClickHouse") and x.endswith("build check"), ), CheckDescription( "Compatibility check", "Checks that clickhouse binary runs on distributions with old libc " "versions. If it fails, ask a maintainer for help", lambda x: x.startswith("Compatibility check"), ), CheckDescription( "Docker image for servers", "The check to build and optionally push the mentioned image to docker hub", lambda x: x.startswith("Docker image") and (x.endswith("building check") or x.endswith("build and push")), ), CheckDescription( "Docs Check", "Builds and tests the documentation", lambda x: x == "Docs Check" ), CheckDescription( "Fast test", "Normally this is the first check that is ran for a PR. It builds ClickHouse " 'and runs most of stateless functional tests, ' "omitting some. If it fails, further checks are not started until it is fixed. " "Look at the report to see which tests fail, then reproduce the failure " 'locally as described here', lambda x: x == "Fast test", ), CheckDescription( "Flaky tests", "Checks if new added or modified tests are flaky by running them repeatedly, " "in parallel, with more randomization. Functional tests are run 100 times " "with address sanitizer, and additional randomization of thread scheduling. " "Integrational tests are run up to 10 times. If at least once a new test has " "failed, or was too long, this check will be red. We don't allow flaky tests, " 'read the doc', lambda x: "tests flaky check" in x, ), CheckDescription( "Install packages", "Checks that the built packages are installable in a clear environment", lambda x: x.startswith("Install packages ("), ), CheckDescription( "Integration tests", "The integration tests report. In parenthesis the package type is given, " "and in square brackets are the optional part/total tests", lambda x: x.startswith("Integration tests ("), ), CheckDescription( "Mergeable Check", "Checks if all other necessary checks are successful", lambda x: x == "Mergeable Check", ), CheckDescription( "Performance Comparison", "Measure changes in query performance. The performance test report is " 'described in detail here. ' "In square brackets are the optional part/total tests", lambda x: x.startswith("Performance Comparison"), ), CheckDescription( "Push to Dockerhub", "The check for building and pushing the CI related docker images to docker hub", lambda x: x.startswith("Push") and "to Dockerhub" in x, ), CheckDescription( "Sqllogic", "Run clickhouse on the " 'sqllogic ' "test set against sqlite and checks that all statements are passed", lambda x: x.startswith("Sqllogic test"), ), CheckDescription( "SQLancer", "Fuzzing tests that detect logical bugs with " 'SQLancer tool', lambda x: x.startswith("SQLancer"), ), CheckDescription( "Stateful tests", "Runs stateful functional tests for ClickHouse binaries built in various " "configurations -- release, debug, with sanitizers, etc", lambda x: x.startswith("Stateful tests ("), ), CheckDescription( "Stateless tests", "Runs stateless functional tests for ClickHouse binaries built in various " "configurations -- release, debug, with sanitizers, etc", lambda x: x.startswith("Stateless tests ("), ), CheckDescription( "Stress test", "Runs stateless functional tests concurrently from several clients to detect " "concurrency-related errors", lambda x: x.startswith("Stress test ("), ), CheckDescription( "Style Check", "Runs a set of checks to keep the code style clean. If some of tests failed, " "see the related log from the report", lambda x: x == "Style Check", ), CheckDescription( "Unit tests", "Runs the unit tests for different release types", lambda x: x.startswith("Unit tests ("), ), CheckDescription( "Upgrade check", "Runs stress tests on server version from last release and then tries to " "upgrade it to the version from the PR. It checks if the new server can " "successfully startup without any errors, crashes or sanitizer asserts", lambda x: x.startswith("Upgrade check ("), ), CheckDescription( "Falback for unknown", "There's no description for the check yet, please add it to " "tests/ci/ci_config.py:CHECK_DESCRIPTIONS", lambda x: True, ), ]