thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-15 02:41:59 +00:00
Code Issues Packages Projects Releases Wiki Activity
2313b7ca78
ClickHouse/tests/ci/ci_config.py

808 lines
34 KiB
Python
Raw Blame History

import random
import re
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
from typing import Dict, List, Optional
from ci_definitions import * # pylint:disable=unused-wildcard-import
from ci_utils import Utils
class CI:
"""
Contains configs for all jobs in the CI pipeline
each config item in the below dicts should be an instance of JobConfig class or inherited from it
"""
MAX_TOTAL_FAILURES_PER_JOB_BEFORE_BLOCKING_CI = 2
# reimport types to CI class so that they visible as CI.* and mypy is happy
# pylint:disable=useless-import-alias,reimported,import-outside-toplevel
from ci_definitions import MQ_JOBS as MQ_JOBS
from ci_definitions import REQUIRED_CHECKS as REQUIRED_CHECKS
from ci_definitions import BuildConfig as BuildConfig
from ci_definitions import BuildNames as BuildNames
from ci_definitions import DigestConfig as DigestConfig
from ci_definitions import JobConfig as JobConfig
from ci_definitions import JobNames as JobNames
from ci_definitions import Labels as Labels
from ci_definitions import Runners as Runners
from ci_definitions import StatusNames as StatusNames
from ci_definitions import SyncState as SyncState
from ci_definitions import Tags as Tags
from ci_definitions import WorkFlowNames as WorkFlowNames
from ci_definitions import WorkflowStages as WorkflowStages
from ci_utils import GH as GH
from ci_utils import Envs as Envs
from ci_utils import Shell as Shell
from ci_utils import Utils as Utils
# Jobs that run for doc related updates
_DOCS_CHECK_JOBS = [JobNames.DOCS_CHECK, JobNames.STYLE_CHECK]
WORKFLOW_CONFIGS = {
WorkFlowNames.JEPSEN: LabelConfig(
run_jobs=[
BuildNames.BINARY_RELEASE,
JobNames.JEPSEN_KEEPER,
JobNames.JEPSEN_SERVER,
]
),
WorkFlowNames.NIGHTLY: LabelConfig(
run_jobs=[
BuildNames.FUZZERS,
JobNames.LIBFUZZER_TEST,
]
),
} # type: Dict[str, LabelConfig]
TAG_CONFIGS = {
Tags.DO_NOT_TEST_LABEL: LabelConfig(run_jobs=[JobNames.STYLE_CHECK]),
Tags.CI_SET_AARCH64: LabelConfig(
run_jobs=[
JobNames.STYLE_CHECK,
BuildNames.PACKAGE_AARCH64,
JobNames.INTEGRATION_TEST_AARCH64,
]
),
Tags.CI_SET_REQUIRED: LabelConfig(
run_jobs=REQUIRED_CHECKS
+ [build for build in BuildNames if build != BuildNames.FUZZERS]
),
Tags.CI_SET_BUILDS: LabelConfig(
run_jobs=[JobNames.STYLE_CHECK, JobNames.BUILD_CHECK]
+ [build for build in BuildNames if build != BuildNames.FUZZERS]
),
Tags.CI_SET_SYNC: LabelConfig(
run_jobs=[
BuildNames.PACKAGE_ASAN,
JobNames.STYLE_CHECK,
JobNames.BUILD_CHECK,
JobNames.UNIT_TEST_ASAN,
JobNames.STATEFUL_TEST_ASAN,
]
),
} # type: Dict[str, LabelConfig]
JOB_CONFIGS: Dict[str, JobConfig] = {
BuildNames.PACKAGE_RELEASE: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.PACKAGE_RELEASE,
compiler="clang-18",
package_type="deb",
static_binary_name="amd64",
additional_pkgs=True,
)
),
BuildNames.PACKAGE_AARCH64: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.PACKAGE_AARCH64,
compiler="clang-18-aarch64",
package_type="deb",
static_binary_name="aarch64",
additional_pkgs=True,
),
runner_type=Runners.BUILDER_AARCH64,
),
BuildNames.PACKAGE_AARCH64_ASAN: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.PACKAGE_AARCH64_ASAN,
compiler="clang-18-aarch64",
sanitizer="address",
package_type="deb",
),
runner_type=Runners.BUILDER_AARCH64,
),
BuildNames.PACKAGE_ASAN: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.PACKAGE_ASAN,
compiler="clang-18",
sanitizer="address",
package_type="deb",
),
),
BuildNames.PACKAGE_UBSAN: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.PACKAGE_UBSAN,
compiler="clang-18",
sanitizer="undefined",
package_type="deb",
),
),
BuildNames.PACKAGE_TSAN: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.PACKAGE_TSAN,
compiler="clang-18",
sanitizer="thread",
package_type="deb",
),
),
BuildNames.PACKAGE_MSAN: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.PACKAGE_MSAN,
compiler="clang-18",
sanitizer="memory",
package_type="deb",
),
),
BuildNames.PACKAGE_DEBUG: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.PACKAGE_DEBUG,
compiler="clang-18",
debug_build=True,
package_type="deb",
sparse_checkout=True, # Check that it works with at least one build, see also update-submodules.sh
),
),
BuildNames.PACKAGE_RELEASE_COVERAGE: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.PACKAGE_RELEASE_COVERAGE,
compiler="clang-18",
coverage=True,
package_type="deb",
),
),
BuildNames.BINARY_RELEASE: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.BINARY_RELEASE,
compiler="clang-18",
package_type="binary",
),
),
BuildNames.BINARY_TIDY: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.BINARY_TIDY,
compiler="clang-18",
debug_build=True,
package_type="binary",
static_binary_name="debug-amd64",
tidy=True,
comment="clang-tidy is used for static analysis",
),
timeout=14400,
),
BuildNames.BINARY_DARWIN: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.BINARY_DARWIN,
compiler="clang-18-darwin",
package_type="binary",
static_binary_name="macos",
),
),
BuildNames.BINARY_AARCH64: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.BINARY_AARCH64,
compiler="clang-18-aarch64",
package_type="binary",
),
),
BuildNames.BINARY_AARCH64_V80COMPAT: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.BINARY_AARCH64_V80COMPAT,
compiler="clang-18-aarch64-v80compat",
package_type="binary",
static_binary_name="aarch64v80compat",
comment="For ARMv8.1 and older",
),
),
BuildNames.BINARY_FREEBSD: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.BINARY_FREEBSD,
compiler="clang-18-freebsd",
package_type="binary",
static_binary_name="freebsd",
),
),
BuildNames.BINARY_DARWIN_AARCH64: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.BINARY_DARWIN_AARCH64,
compiler="clang-18-darwin-aarch64",
package_type="binary",
static_binary_name="macos-aarch64",
),
),
BuildNames.BINARY_PPC64LE: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.BINARY_PPC64LE,
compiler="clang-18-ppc64le",
package_type="binary",
static_binary_name="powerpc64le",
),
),
BuildNames.BINARY_AMD64_COMPAT: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.BINARY_AMD64_COMPAT,
compiler="clang-18-amd64-compat",
package_type="binary",
static_binary_name="amd64compat",
comment="SSE2-only build",
),
),
BuildNames.BINARY_AMD64_MUSL: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.BINARY_AMD64_MUSL,
compiler="clang-18-amd64-musl",
package_type="binary",
static_binary_name="amd64musl",
comment="Build with Musl",
),
),
BuildNames.BINARY_RISCV64: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.BINARY_RISCV64,
compiler="clang-18-riscv64",
package_type="binary",
static_binary_name="riscv64",
),
),
BuildNames.BINARY_S390X: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.BINARY_S390X,
compiler="clang-18-s390x",
package_type="binary",
static_binary_name="s390x",
),
),
BuildNames.BINARY_LOONGARCH64: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.BINARY_LOONGARCH64,
compiler="clang-18-loongarch64",
package_type="binary",
static_binary_name="loongarch64",
),
),
BuildNames.FUZZERS: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.FUZZERS,
compiler="clang-18",
package_type="fuzzers",
),
run_by_labels=[Tags.libFuzzer],
),
BuildNames.FUZZERS_ASAN: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.FUZZERS_ASAN,
compiler="clang-18",
sanitizer="address",
package_type="fuzzers",
),
run_by_labels=[Tags.libFuzzer],
),
BuildNames.FUZZERS_UBSAN: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.FUZZERS_UBSAN,
compiler="clang-18",
sanitizer="undefined",
package_type="fuzzers",
),
run_by_labels=[Tags.libFuzzer],
),
BuildNames.FUZZERS_TSAN: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.FUZZERS_TSAN,
compiler="clang-18",
sanitizer="thread",
package_type="fuzzers",
),
run_by_labels=[Tags.libFuzzer],
),
BuildNames.FUZZERS_MSAN: CommonJobConfigs.BUILD.with_properties(
build_config=BuildConfig(
name=BuildNames.FUZZERS_MSAN,
compiler="clang-18",
sanitizer="memory",
package_type="fuzzers",
),
run_by_labels=[Tags.libFuzzer],
),
JobNames.BUILD_CHECK: CommonJobConfigs.BUILD_REPORT.with_properties(),
JobNames.INSTALL_TEST_AMD: CommonJobConfigs.INSTALL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE]
),
JobNames.INSTALL_TEST_AARCH64: CommonJobConfigs.INSTALL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_AARCH64],
runner_type=Runners.STYLE_CHECKER_AARCH64,
),
JobNames.STATEFUL_TEST_ASAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_ASAN]
),
JobNames.STATEFUL_TEST_AARCH64_ASAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_AARCH64_ASAN],
runner_type=Runners.FUNC_TESTER_AARCH64,
),
JobNames.STATEFUL_TEST_TSAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_TSAN]
),
JobNames.STATEFUL_TEST_MSAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_MSAN]
),
JobNames.STATEFUL_TEST_UBSAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_UBSAN]
),
JobNames.STATEFUL_TEST_DEBUG: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_DEBUG]
),
JobNames.STATEFUL_TEST_RELEASE: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE]
),
JobNames.STATEFUL_TEST_RELEASE_COVERAGE: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE_COVERAGE]
),
JobNames.STATEFUL_TEST_AARCH64: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_AARCH64],
runner_type=Runners.FUNC_TESTER_AARCH64,
),
JobNames.STATEFUL_TEST_PARALLEL_REPL_RELEASE: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE]
),
JobNames.STATEFUL_TEST_PARALLEL_REPL_DEBUG: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_DEBUG]
),
JobNames.STATEFUL_TEST_PARALLEL_REPL_ASAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_ASAN],
random_bucket="parrepl_with_sanitizer",
),
JobNames.STATEFUL_TEST_PARALLEL_REPL_MSAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_MSAN],
random_bucket="parrepl_with_sanitizer",
),
JobNames.STATEFUL_TEST_PARALLEL_REPL_UBSAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_UBSAN],
random_bucket="parrepl_with_sanitizer",
),
JobNames.STATEFUL_TEST_PARALLEL_REPL_TSAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_TSAN],
random_bucket="parrepl_with_sanitizer",
timeout=3600,
),
JobNames.STATELESS_TEST_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_ASAN], num_batches=2
),
JobNames.STATELESS_TEST_AARCH64_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_AARCH64_ASAN],
num_batches=2,
runner_type=Runners.FUNC_TESTER_AARCH64,
),
JobNames.STATELESS_TEST_TSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_TSAN], num_batches=4
),
JobNames.STATELESS_TEST_MSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_MSAN], num_batches=4
),
JobNames.STATELESS_TEST_UBSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_UBSAN], num_batches=2
),
JobNames.STATELESS_TEST_DEBUG: CommonJobConfigs.STATELESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=2
),
JobNames.STATELESS_TEST_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE],
),
JobNames.STATELESS_TEST_RELEASE_COVERAGE: CommonJobConfigs.STATELESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE_COVERAGE], num_batches=6
),
JobNames.STATELESS_TEST_AARCH64: CommonJobConfigs.STATELESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_AARCH64],
runner_type=Runners.FUNC_TESTER_AARCH64,
),
JobNames.STATELESS_TEST_OLD_ANALYZER_S3_REPLICATED_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=2
),
JobNames.STATELESS_TEST_S3_DEBUG: CommonJobConfigs.STATELESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=1
),
JobNames.STATELESS_TEST_AZURE_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_ASAN], num_batches=3, release_only=True
),
JobNames.STATELESS_TEST_S3_TSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_TSAN],
num_batches=3,
),
JobNames.STRESS_TEST_DEBUG: CommonJobConfigs.STRESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_DEBUG],
),
JobNames.STRESS_TEST_TSAN: CommonJobConfigs.STRESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_TSAN],
),
JobNames.STRESS_TEST_ASAN: CommonJobConfigs.STRESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_ASAN],
random_bucket="stress_with_sanitizer",
),
JobNames.STRESS_TEST_UBSAN: CommonJobConfigs.STRESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_UBSAN],
random_bucket="stress_with_sanitizer",
),
JobNames.STRESS_TEST_MSAN: CommonJobConfigs.STRESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_MSAN],
random_bucket="stress_with_sanitizer",
),
JobNames.STRESS_TEST_AZURE_TSAN: CommonJobConfigs.STRESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_TSAN], release_only=True
),
JobNames.STRESS_TEST_AZURE_MSAN: CommonJobConfigs.STRESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_MSAN], release_only=True
),
JobNames.UPGRADE_TEST_ASAN: CommonJobConfigs.UPGRADE_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_ASAN],
random_bucket="upgrade_with_sanitizer",
pr_only=True,
),
JobNames.UPGRADE_TEST_TSAN: CommonJobConfigs.UPGRADE_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_TSAN],
random_bucket="upgrade_with_sanitizer",
pr_only=True,
),
JobNames.UPGRADE_TEST_MSAN: CommonJobConfigs.UPGRADE_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_MSAN],
random_bucket="upgrade_with_sanitizer",
pr_only=True,
),
JobNames.UPGRADE_TEST_DEBUG: CommonJobConfigs.UPGRADE_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_DEBUG], pr_only=True
),
JobNames.INTEGRATION_TEST_ASAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_ASAN],
release_only=True,
num_batches=4,
timeout=10800,
),
JobNames.INTEGRATION_TEST_ASAN_OLD_ANALYZER: CommonJobConfigs.INTEGRATION_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_ASAN],
num_batches=6,
),
JobNames.INTEGRATION_TEST_TSAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_TSAN],
num_batches=6,
timeout=9000, # the job timed out with default value (7200)
),
JobNames.INTEGRATION_TEST_AARCH64: CommonJobConfigs.INTEGRATION_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_AARCH64],
num_batches=6,
runner_type=Runners.FUNC_TESTER_AARCH64,
),
JobNames.INTEGRATION_TEST: CommonJobConfigs.INTEGRATION_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE],
num_batches=4,
release_only=True,
),
JobNames.INTEGRATION_TEST_FLAKY: CommonJobConfigs.INTEGRATION_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_ASAN],
pr_only=True,
# TODO: approach with reference job names does not work because digest may not be calculated if job skipped in wf
# reference_job_name=JobNames.INTEGRATION_TEST_TSAN,
timeout=4 * 3600, # to be able to process many updated tests
),
JobNames.COMPATIBILITY_TEST: CommonJobConfigs.COMPATIBILITY_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE],
required_on_release_branch=True,
),
JobNames.COMPATIBILITY_TEST_AARCH64: CommonJobConfigs.COMPATIBILITY_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_AARCH64],
required_on_release_branch=True,
runner_type=Runners.STYLE_CHECKER_AARCH64,
),
JobNames.UNIT_TEST: CommonJobConfigs.UNIT_TEST.with_properties(
required_builds=[BuildNames.BINARY_RELEASE],
),
JobNames.UNIT_TEST_ASAN: CommonJobConfigs.UNIT_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_ASAN],
),
JobNames.UNIT_TEST_MSAN: CommonJobConfigs.UNIT_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_MSAN],
),
JobNames.UNIT_TEST_TSAN: CommonJobConfigs.UNIT_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_TSAN],
),
JobNames.UNIT_TEST_UBSAN: CommonJobConfigs.UNIT_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_UBSAN],
),
JobNames.AST_FUZZER_TEST_DEBUG: CommonJobConfigs.ASTFUZZER_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_DEBUG],
),
JobNames.AST_FUZZER_TEST_ASAN: CommonJobConfigs.ASTFUZZER_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_ASAN],
),
JobNames.AST_FUZZER_TEST_MSAN: CommonJobConfigs.ASTFUZZER_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_MSAN],
),
JobNames.AST_FUZZER_TEST_TSAN: CommonJobConfigs.ASTFUZZER_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_TSAN],
),
JobNames.AST_FUZZER_TEST_UBSAN: CommonJobConfigs.ASTFUZZER_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_UBSAN],
),
JobNames.STATELESS_TEST_FLAKY_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_ASAN],
pr_only=True,
timeout=3 * 3600,
# TODO: approach with reference job names does not work because digest may not be calculated if job skipped in wf
# reference_job_name=JobNames.STATELESS_TEST_RELEASE,
),
JobNames.JEPSEN_KEEPER: JobConfig(
required_builds=[BuildNames.BINARY_RELEASE],
run_by_labels=[Labels.JEPSEN_TEST],
run_command="jepsen_check.py keeper",
runner_type=Runners.STYLE_CHECKER_AARCH64,
),
JobNames.JEPSEN_SERVER: JobConfig(
required_builds=[BuildNames.BINARY_RELEASE],
run_by_labels=[Labels.JEPSEN_TEST],
run_command="jepsen_check.py server",
runner_type=Runners.STYLE_CHECKER_AARCH64,
),
JobNames.PERFORMANCE_TEST_AMD64: CommonJobConfigs.PERF_TESTS.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=4
),
JobNames.PERFORMANCE_TEST_AARCH64: CommonJobConfigs.PERF_TESTS.with_properties(
required_builds=[BuildNames.PACKAGE_AARCH64],
num_batches=4,
run_by_labels=[Labels.PR_PERFORMANCE],
runner_type=Runners.FUNC_TESTER_AARCH64,
),
JobNames.SQLANCER: CommonJobConfigs.SQLLANCER_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE],
),
JobNames.SQLANCER_DEBUG: CommonJobConfigs.SQLLANCER_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_DEBUG],
),
# TODO: job does not work at all, uncomment and fix
# JobNames.SQL_LOGIC_TEST: CommonJobConfigs.SQLLOGIC_TEST.with_properties(
# required_builds=[BuildNames.PACKAGE_RELEASE],
# ),
JobNames.SQLTEST: CommonJobConfigs.SQL_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE],
),
JobNames.CLICKBENCH_TEST: CommonJobConfigs.CLICKBENCH_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE],
),
JobNames.CLICKBENCH_TEST_AARCH64: CommonJobConfigs.CLICKBENCH_TEST.with_properties(
required_builds=[BuildNames.PACKAGE_AARCH64],
runner_type=Runners.FUNC_TESTER_AARCH64,
),
JobNames.LIBFUZZER_TEST: JobConfig(
required_builds=[BuildNames.FUZZERS],
run_by_labels=[Tags.libFuzzer],
timeout=5400,
run_command='libfuzzer_test_check.py "$CHECK_NAME"',
runner_type=Runners.FUNC_TESTER,
),
JobNames.DOCKER_SERVER: CommonJobConfigs.DOCKER_SERVER.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE, BuildNames.PACKAGE_AARCH64]
),
JobNames.DOCKER_KEEPER: CommonJobConfigs.DOCKER_SERVER.with_properties(
required_builds=[BuildNames.PACKAGE_RELEASE, BuildNames.PACKAGE_AARCH64]
),
JobNames.DOCS_CHECK: JobConfig(
digest=DigestConfig(
include_paths=["**/*.md", "./docs", "tests/ci/docs_check.py"],
docker=["clickhouse/docs-builder"],
),
run_command="docs_check.py",
runner_type=Runners.FUNC_TESTER,
),
JobNames.FAST_TEST: JobConfig(
pr_only=True,
digest=DigestConfig(
include_paths=[
"./tests/queries/0_stateless/",
"./tests/docker_scripts/",
"./tests/config/",
"./tests/clickhouse-test",
],
exclude_files=[".md"],
docker=["clickhouse/fasttest"],
),
timeout=2400,
runner_type=Runners.BUILDER,
),
JobNames.STYLE_CHECK: JobConfig(
run_always=True,
runner_type=Runners.STYLE_CHECKER_AARCH64,
),
JobNames.BUGFIX_VALIDATE: JobConfig(
run_by_labels=[Labels.PR_BUGFIX, Labels.PR_CRITICAL_BUGFIX],
run_command="bugfix_validate_check.py",
timeout=2400,
runner_type=Runners.STYLE_CHECKER,
),
}
@classmethod
def get_tag_config(cls, label_name: str) -> Optional[LabelConfig]:
for label, config in cls.TAG_CONFIGS.items():
if Utils.normalize_string(label_name) == Utils.normalize_string(label):
return config
return None
@classmethod
def get_job_ci_stage(cls, job_name: str, non_blocking_ci: bool = False) -> str:
if job_name in [
JobNames.STYLE_CHECK,
JobNames.FAST_TEST,
JobNames.JEPSEN_SERVER,
JobNames.JEPSEN_KEEPER,
JobNames.BUILD_CHECK,
]:
return WorkflowStages.NA
stage_type = None
if cls.is_build_job(job_name):
for _job, config in cls.JOB_CONFIGS.items():
if config.required_builds and job_name in config.required_builds:
stage_type = WorkflowStages.BUILDS_1
break
else:
stage_type = WorkflowStages.BUILDS_2
elif cls.is_docs_job(job_name):
stage_type = WorkflowStages.TESTS_1
elif cls.is_test_job(job_name):
if job_name in CI.JOB_CONFIGS:
if job_name in REQUIRED_CHECKS:
stage_type = WorkflowStages.TESTS_1
else:
stage_type = WorkflowStages.TESTS_2
assert stage_type, f"BUG [{job_name}]"
if non_blocking_ci and stage_type == WorkflowStages.TESTS_2:
stage_type = WorkflowStages.TESTS_2_WW
return stage_type
@classmethod
def get_job_config(cls, check_name: str) -> JobConfig:
return cls.JOB_CONFIGS[check_name]
@classmethod
def get_required_build_name(cls, check_name: str) -> str:
assert check_name in cls.JOB_CONFIGS
required_builds = cls.JOB_CONFIGS[check_name].required_builds
assert required_builds and len(required_builds) == 1
return required_builds[0]
@classmethod
def get_job_parents(cls, check_name: str) -> List[str]:
return cls.JOB_CONFIGS[check_name].required_builds or []
@classmethod
def get_workflow_jobs_with_configs(
cls,
is_mq: bool,
is_docs_only: bool,
is_master: bool,
is_pr: bool,
workflow_name: str,
) -> Dict[str, JobConfig]:
"""
get a list of all jobs for a workflow with configs
"""
if is_mq:
jobs = MQ_JOBS
elif is_docs_only:
jobs = cls._DOCS_CHECK_JOBS
elif workflow_name:
assert (
workflow_name in cls.WORKFLOW_CONFIGS
), "Workflow name if provided must be configured in WORKFLOW_CONFIGS"
jobs = list(cls.WORKFLOW_CONFIGS[workflow_name].run_jobs)
else:
# add all jobs
jobs = list(cls.JOB_CONFIGS)
if is_master:
for job in MQ_JOBS:
jobs.remove(job)
randomization_bucket_jobs = {} # type: Dict[str, Dict[str, JobConfig]]
res = {} # type: Dict[str, JobConfig]
for job in jobs:
job_config = cls.JOB_CONFIGS[job]
if job_config.random_bucket and is_pr:
if job_config.random_bucket not in randomization_bucket_jobs:
randomization_bucket_jobs[job_config.random_bucket] = {}
randomization_bucket_jobs[job_config.random_bucket][job] = job_config
continue
res[job] = job_config
# add to the result a random job from each random bucket, if any
for bucket, jobs_configs in randomization_bucket_jobs.items():
job = random.choice(list(jobs_configs))
print(f"Pick job [{job}] from randomization bucket [{bucket}]")
res[job] = jobs_configs[job]
return res
@classmethod
def is_build_job(cls, job: str) -> bool:
return job in cls.BuildNames
@classmethod
def is_test_job(cls, job: str) -> bool:
return not cls.is_build_job(job)
@classmethod
def is_docs_job(cls, job: str) -> bool:
return job == JobNames.DOCS_CHECK
@classmethod
def is_required(cls, check_name: str) -> bool:
"""Checks if a check_name is in REQUIRED_CHECKS, including batched jobs"""
_BATCH_REGEXP = re.compile(r"\s+\[[0-9/]+\]$")
if check_name in REQUIRED_CHECKS:
return True
if batch := _BATCH_REGEXP.search(check_name):
return check_name[: batch.start()] in REQUIRED_CHECKS
return False
@classmethod
def get_build_config(cls, build_name: str) -> BuildConfig:
assert build_name in cls.JOB_CONFIGS, f"Invalid build name [{build_name}]"
res = cls.JOB_CONFIGS[build_name].build_config
assert res, f"not a build [{build_name}] or invalid JobConfig"
return res
@classmethod
def is_workflow_ok(cls) -> bool:
# TODO: temporary method to make Mergeable check working
res = cls.GH.get_workflow_results()
if not res:
print("ERROR: no workflow results found")
return False
for workflow_job, workflow_data in res.items():
status = workflow_data["result"]
if status in (
cls.GH.ActionStatuses.SUCCESS,
cls.GH.ActionStatuses.SKIPPED,
):
print(f"Workflow status for [{workflow_job}] is [{status}] - continue")
elif status in (cls.GH.ActionStatuses.FAILURE,):
if workflow_job in (
WorkflowStages.TESTS_2,
WorkflowStages.TESTS_2_WW,
):
print(
f"Failed Workflow status for [{workflow_job}], it's not required - continue"
)
continue
print(f"Failed Workflow status for [{workflow_job}]")
return False
return True
if __name__ == "__main__":
parser = ArgumentParser(
formatter_class=ArgumentDefaultsHelpFormatter,
description="The script provides build config for GITHUB_ENV or shell export",
)
parser.add_argument("--build-name", help="the build config to export")
parser.add_argument(
"--export",
action="store_true",
help="if set, the ENV parameters are provided for shell export",
)
args = parser.parse_args()
assert (
args.build_name in CI.JOB_CONFIGS
), f"Build name [{args.build_name}] is not valid"
build_config = CI.JOB_CONFIGS[args.build_name].build_config
assert build_config, "--export must not be used for non-build jobs"
print(build_config.export_env(args.export))
Reference in New Issue View Git Blame Copy Permalink