diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 8783f959ec6..c065219f980 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -13,9 +13,7 @@ on: # yamllint disable-line rule:truthy - opened branches: - master -########################################################################################## -##################################### SMALL CHECKS ####################################### -########################################################################################## + jobs: RunConfig: runs-on: [self-hosted, style-checker-aarch64] @@ -70,13 +68,13 @@ jobs: python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --post --job-name 'Style check' BuildDockers: needs: [RunConfig] - if: ${{ !failure() && !cancelled() }} + if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }} uses: ./.github/workflows/reusable_docker.yml with: data: ${{ needs.RunConfig.outputs.data }} StyleCheck: needs: [RunConfig, BuildDockers] - if: ${{ !failure() && !cancelled() }} + if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Style check')}} uses: ./.github/workflows/reusable_test.yml with: test_name: Style check @@ -89,19 +87,9 @@ jobs: ROBOT_CLICKHOUSE_SSH_KEY< Dict[str, Dict[str, Any]]: + """ + populates GH Actions' workflow with real jobs + "Builds_1": [{"job_name": NAME, "runner_type": RUNER_TYPE}] + "Tests_1": [{"job_name": NAME, "runner_type": RUNER_TYPE}] + ... + """ + result = {} # type: Dict[str, Any] + stages_to_do = [] + for job in jobs_data["jobs_to_do"]: + stage_type = CI_CONFIG.get_job_ci_stage(job) + if stage_type == CIStages.NA: + continue + if stage_type not in result: + result[stage_type] = [] + stages_to_do.append(stage_type) + result[stage_type].append( + {"job_name": job, "runner_type": CI_CONFIG.get_runner_type(job)} + ) + result["stages_to_do"] = stages_to_do + return result + + def _create_gh_status( commit: Any, job: str, batch: int, num_batches: int, job_status: CommitStatusData ) -> None: @@ -1734,6 +1757,7 @@ def main() -> int: result["build"] = build_digest result["docs"] = docs_digest result["ci_flags"] = ci_flags + result["stages_data"] = _generate_ci_stage_config(jobs_data) result["jobs_data"] = jobs_data result["docker_data"] = docker_data ### CONFIGURE action: end diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 21749d16069..2a971161e75 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +from copy import deepcopy import logging from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser from dataclasses import dataclass, field @@ -10,6 +11,24 @@ from ci_utils import WithIter from integration_test_images import IMAGES +class CIStages: + NA = "UNKNOWN" + BUILDS_1 = "Builds_1" + BUILDS_2 = "Builds_2" + TESTS_1 = "Tests_1" + TESTS_2 = "Tests_2" + + +class Runners(metaclass=WithIter): + BUILDER = "builder" + STYLE_CHECKER = "style-checker" + STYLE_CHECKER_ARM = "style-checker-aarch64" + FUNC_TESTER = "func-tester" + FUNC_TESTER_ARM = "func-tester-aarch64" + STRESS_TESTER = "stress-tester" + FUZZER_UNIT_TESTER = "fuzzer-unit-tester" + + class Labels(metaclass=WithIter): """ Label names or commit tokens in normalized form @@ -207,6 +226,45 @@ class JobConfig: random_bucket: str = "" +builds_job_config = JobConfig( + required_on_release_branch=True, + digest=DigestConfig( + include_paths=[ + "./src", + "./contrib/*-cmake", + "./contrib/consistent-hashing", + "./contrib/murmurhash", + "./contrib/libfarmhash", + "./contrib/pdqsort", + "./contrib/cityhash102", + "./contrib/sparse-checkout", + "./contrib/libmetrohash", + "./contrib/update-submodules.sh", + "./contrib/CMakeLists.txt", + "./CMakeLists.txt", + "./PreLoad.cmake", + "./cmake", + "./base", + "./programs", + "./packages", + "./docker/packager/packager", + "./rust", + # FIXME: This is a WA to rebuild the CH and recreate the Performance.tar.zst artifact + # when there are changes in performance test scripts. + # Due to the current design of the perf test we need to rebuild CH when the performance test changes, + # otherwise the changes will not be visible in the PerformanceTest job in CI + "./tests/performance", + ], + exclude_files=[".md"], + docker=["clickhouse/binary-builder"], + git_submodules=True, + ), + run_command="build_check.py $BUILD_NAME", +) +fuzzer_build_job_config = deepcopy(builds_job_config) +fuzzer_build_job_config.run_by_label = Labels.libFuzzer + + @dataclass class BuildConfig: name: str @@ -222,43 +280,7 @@ class BuildConfig: sparse_checkout: bool = False comment: str = "" static_binary_name: str = "" - job_config: JobConfig = field( - default_factory=lambda: JobConfig( - required_on_release_branch=True, - digest=DigestConfig( - include_paths=[ - "./src", - "./contrib/*-cmake", - "./contrib/consistent-hashing", - "./contrib/murmurhash", - "./contrib/libfarmhash", - "./contrib/pdqsort", - "./contrib/cityhash102", - "./contrib/sparse-checkout", - "./contrib/libmetrohash", - "./contrib/update-submodules.sh", - "./contrib/CMakeLists.txt", - "./CMakeLists.txt", - "./PreLoad.cmake", - "./cmake", - "./base", - "./programs", - "./packages", - "./docker/packager/packager", - "./rust", - # FIXME: This is a WA to rebuild the CH and recreate the Performance.tar.zst artifact - # when there are changes in performance test scripts. - # Due to the current design of the perf test we need to rebuild CH when the performance test changes, - # otherwise the changes will not be visible in the PerformanceTest job in CI - "./tests/performance", - ], - exclude_files=[".md"], - docker=["clickhouse/binary-builder"], - git_submodules=True, - ), - run_command="build_check.py $BUILD_NAME", - ) - ) + job_config: JobConfig = field(default_factory=lambda: deepcopy(builds_job_config)) def export_env(self, export: bool = False) -> str: def process(field_name: str, field: Union[bool, str]) -> str: @@ -464,6 +486,19 @@ sql_test_params = { "run_command": "sqltest.py", "timeout": 10800, } +clickbench_test_params = { + "digest": DigestConfig( + include_paths=[ + "tests/ci/clickbench.py", + ], + docker=["clickhouse/clickbench"], + ), + "run_command": 'clickbench.py "$CHECK_NAME"', +} +install_test_params = { + "digest": install_check_digest, + "run_command": 'install_check.py "$CHECK_NAME"', +} @dataclass @@ -485,6 +520,37 @@ class CIConfig: return config return None + def get_job_ci_stage(self, job_name: str) -> str: + if job_name in [ + JobNames.STYLE_CHECK, + JobNames.FAST_TEST, + JobNames.JEPSEN_KEEPER, + JobNames.BUILD_CHECK, + JobNames.BUILD_CHECK_SPECIAL, + ]: + # FIXME: we can't currently handle Jepsen in the Stage as it's job has concurrency directive + # BUILD_CHECK and BUILD_CHECK_SPECIAL runs not in stage because we need them even if Builds stage failed + return CIStages.NA + stage_type = None + if self.is_build_job(job_name): + stage_type = CIStages.BUILDS_1 + if job_name in CI_CONFIG.get_builds_for_report( + JobNames.BUILD_CHECK_SPECIAL + ): + # special builds go to Build_2 stage to not delay Builds_1/Test_1 + stage_type = CIStages.BUILDS_2 + elif self.is_docs_job(job_name): + stage_type = CIStages.TESTS_1 + elif job_name == JobNames.BUILD_CHECK_SPECIAL: + stage_type = CIStages.TESTS_2 + elif self.is_test_job(job_name): + stage_type = CIStages.TESTS_1 + if job_name == JobNames.LIBFUZZER_TEST: + # since fuzzers build in Builds_2, test must be in Tests_2 + stage_type = CIStages.TESTS_2 + assert stage_type, f"BUG [{job_name}]" + return stage_type + def get_job_config(self, check_name: str) -> JobConfig: res = None for config in ( @@ -498,6 +564,63 @@ class CIConfig: break return res # type: ignore + def get_runner_type(self, check_name: str) -> str: + result = None + if self.is_build_job(check_name) or check_name == JobNames.FAST_TEST: + result = Runners.BUILDER + elif any( + words in check_name.lower() + for words in [ + "install packages", + "compatibility check", + "docker", + "build check", + "jepsen", + "style check", + ] + ): + result = Runners.STYLE_CHECKER + elif check_name == JobNames.DOCS_CHECK: + # docs job is demanding + result = Runners.FUNC_TESTER_ARM + elif any( + words in check_name.lower() + for words in [ + "stateless", + "stateful", + "clickbench", + "sqllogic test", + "libfuzzer", + "bugfix validation", + ] + ): + result = Runners.FUNC_TESTER + elif any( + words in check_name.lower() + for words in ["stress", "upgrade", "integration", "performance comparison"] + ): + result = Runners.STRESS_TESTER + elif any( + words in check_name.lower() + for words in ["ast fuzzer", "unit tests", "sqlancer", "sqltest"] + ): + result = Runners.FUZZER_UNIT_TESTER + + assert result, f"BUG, no runner for [{check_name}]" + + if ("aarch" in check_name or "arm" in check_name) and "aarch" not in result: + if result == Runners.STRESS_TESTER: + # FIXME: no arm stress tester group atm + result = Runners.FUNC_TESTER_ARM + elif result == Runners.BUILDER: + # crosscompile - no arm required + pass + else: + # switch to aarch64 runnner + result += "-aarch64" + + return result + @staticmethod def normalize_string(input_string: str) -> str: lowercase_string = input_string.lower() @@ -598,11 +721,7 @@ class CIConfig: @classmethod def is_test_job(cls, job: str) -> bool: - return ( - not cls.is_build_job(job) - and not cls.is_build_job(job) - and job != JobNames.STYLE_CHECK - ) + return not cls.is_build_job(job) and job != JobNames.STYLE_CHECK @classmethod def is_docs_job(cls, job: str) -> bool: @@ -843,7 +962,7 @@ CI_CONFIG = CIConfig( name=Build.FUZZERS, compiler="clang-17", package_type="fuzzers", - job_config=JobConfig(run_by_label=Labels.libFuzzer), + job_config=fuzzer_build_job_config, ), }, builds_report_config={ @@ -887,6 +1006,7 @@ CI_CONFIG = CIConfig( include_paths=["**/*.md", "./docs", "tests/ci/docs_check.py"], docker=["clickhouse/docs-builder"], ), + run_command="docs_check.py", ), ), JobNames.FAST_TEST: TestConfig( @@ -916,10 +1036,10 @@ CI_CONFIG = CIConfig( }, test_configs={ JobNames.INSTALL_TEST_AMD: TestConfig( - Build.PACKAGE_RELEASE, job_config=JobConfig(digest=install_check_digest) + Build.PACKAGE_RELEASE, job_config=JobConfig(**install_test_params) # type: ignore ), JobNames.INSTALL_TEST_ARM: TestConfig( - Build.PACKAGE_AARCH64, job_config=JobConfig(digest=install_check_digest) + Build.PACKAGE_AARCH64, job_config=JobConfig(**install_test_params) # type: ignore ), JobNames.STATEFUL_TEST_ASAN: TestConfig( Build.PACKAGE_ASAN, job_config=JobConfig(**stateful_test_common_params) # type: ignore @@ -1137,9 +1257,20 @@ CI_CONFIG = CIConfig( JobNames.SQLTEST: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig(**sql_test_params) # type: ignore ), - JobNames.CLCIKBENCH_TEST: TestConfig(Build.PACKAGE_RELEASE), - JobNames.CLCIKBENCH_TEST_ARM: TestConfig(Build.PACKAGE_AARCH64), - JobNames.LIBFUZZER_TEST: TestConfig(Build.FUZZERS, job_config=JobConfig(run_by_label=Labels.libFuzzer)), # type: ignore + JobNames.CLCIKBENCH_TEST: TestConfig( + Build.PACKAGE_RELEASE, job_config=JobConfig(**clickbench_test_params) # type: ignore + ), + JobNames.CLCIKBENCH_TEST_ARM: TestConfig( + Build.PACKAGE_AARCH64, job_config=JobConfig(**clickbench_test_params) # type: ignore + ), + JobNames.LIBFUZZER_TEST: TestConfig( + Build.FUZZERS, + job_config=JobConfig( + run_by_label=Labels.libFuzzer, + timeout=10800, + run_command='libfuzzer_test_check.py "$CHECK_NAME" 10800', + ), + ), # type: ignore }, ) CI_CONFIG.validate() diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py index 49d49d9c328..04c90105276 100644 --- a/tests/ci/test_ci_config.py +++ b/tests/ci/test_ci_config.py @@ -1,15 +1,12 @@ #!/usr/bin/env python3 import unittest +from ci_config import JobNames, CI_CONFIG, Runners class TestCIConfig(unittest.TestCase): - def test_no_errors_in_ci_config(self): - raised = None - try: - from ci_config import ( # pylint: disable=import-outside-toplevel - CI_CONFIG as _, - ) - except Exception as exc: - raised = exc - self.assertIsNone(raised, f"CI_CONFIG import raised error {raised}") + def test_runner_config(self): + """check runner is provided w/o exception""" + for job in JobNames: + runner = CI_CONFIG.get_runner_type(job) + self.assertIn(runner, Runners)