From 59b871663fa0296d5f91d862bedf18915db233d0 Mon Sep 17 00:00:00 2001 From: Max K Date: Mon, 17 Jun 2024 08:33:31 +0200 Subject: [PATCH] More ci unit tests --- tests/ci/ci_config.py | 18 +++++++++++++----- tests/ci/ci_definitions.py | 25 ++++++++++++++++++++++--- tests/ci/test_ci_config.py | 36 ++++++++++++++++++++++++++++++++++++ tests/ci/test_ci_options.py | 14 +++++--------- 4 files changed, 76 insertions(+), 17 deletions(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 655314cb629..60c6a60af1a 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -260,7 +260,8 @@ class CI: required_builds=[BuildNames.PACKAGE_RELEASE] ), JobNames.INSTALL_TEST_ARM: CommonJobConfigs.INSTALL_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_AARCH64] + required_builds=[BuildNames.PACKAGE_AARCH64], + runner_type=Runners.STYLE_CHECKER_ARM, ), JobNames.STATEFUL_TEST_ASAN: CommonJobConfigs.STATEFUL_TEST.with_properties( required_builds=[BuildNames.PACKAGE_ASAN] @@ -284,7 +285,8 @@ class CI: required_builds=[BuildNames.PACKAGE_RELEASE_COVERAGE] ), JobNames.STATEFUL_TEST_AARCH64: CommonJobConfigs.STATEFUL_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_AARCH64] + required_builds=[BuildNames.PACKAGE_AARCH64], + runner_type=Runners.FUNC_TESTER_ARM, ), JobNames.STATEFUL_TEST_PARALLEL_REPL_RELEASE: CommonJobConfigs.STATEFUL_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE] @@ -331,6 +333,7 @@ class CI: ), JobNames.STATELESS_TEST_AARCH64: CommonJobConfigs.STATELESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_AARCH64], + runner_type=Runners.FUNC_TESTER_ARM, ), JobNames.STATELESS_TEST_OLD_ANALYZER_S3_REPLICATED_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=4 @@ -397,7 +400,9 @@ class CI: required_builds=[BuildNames.PACKAGE_TSAN], num_batches=6 ), JobNames.INTEGRATION_TEST_ARM: CommonJobConfigs.INTEGRATION_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_AARCH64], num_batches=6 + required_builds=[BuildNames.PACKAGE_AARCH64], + num_batches=6, + runner_type=Runners.FUNC_TESTER_ARM, ), JobNames.INTEGRATION_TEST: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE], @@ -414,6 +419,7 @@ class CI: JobNames.COMPATIBILITY_TEST_ARM: CommonJobConfigs.COMPATIBILITY_TEST.with_properties( required_builds=[BuildNames.PACKAGE_AARCH64], required_on_release_branch=True, + runner_type=Runners.STYLE_CHECKER_ARM, ), JobNames.UNIT_TEST: CommonJobConfigs.UNIT_TEST.with_properties( required_builds=[BuildNames.BINARY_RELEASE], @@ -467,6 +473,7 @@ class CI: required_builds=[BuildNames.PACKAGE_AARCH64], num_batches=4, run_by_label="pr-performance", + runner_type=Runners.FUNC_TESTER_ARM, ), JobNames.SQLANCER: CommonJobConfigs.SQLLANCER_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE], @@ -480,11 +487,12 @@ class CI: JobNames.SQLTEST: CommonJobConfigs.SQL_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE], ), - JobNames.CLICKBENCH_TEST: CommonJobConfigs.SQL_TEST.with_properties( + JobNames.CLICKBENCH_TEST: CommonJobConfigs.CLICKBENCH_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE], ), - JobNames.CLICKBENCH_TEST_ARM: CommonJobConfigs.SQL_TEST.with_properties( + JobNames.CLICKBENCH_TEST_ARM: CommonJobConfigs.CLICKBENCH_TEST.with_properties( required_builds=[BuildNames.PACKAGE_AARCH64], + runner_type=Runners.FUNC_TESTER_ARM, ), JobNames.LIBFUZZER_TEST: JobConfig( required_builds=[BuildNames.FUZZERS], diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py index 9a4b845a61e..fdd5dc7a671 100644 --- a/tests/ci/ci_definitions.py +++ b/tests/ci/ci_definitions.py @@ -280,6 +280,8 @@ class JobConfig: # GH Runner type (tag from @Runners) runner_type: str + # used for config validation in ci unittests + job_name_keyword: str = "" # builds required for the job (applicable for test jobs) required_builds: Optional[List[str]] = None # build config for the build job (applicable for builds) @@ -328,6 +330,7 @@ class CommonJobConfigs: """ BUILD_REPORT = JobConfig( + job_name_keyword="build_check", run_command="build_report_check.py", digest=DigestConfig( include_paths=[ @@ -338,23 +341,26 @@ class CommonJobConfigs: runner_type=Runners.STYLE_CHECKER_ARM, ) COMPATIBILITY_TEST = JobConfig( + job_name_keyword="compatibility", digest=DigestConfig( include_paths=["./tests/ci/compatibility_check.py"], docker=["clickhouse/test-old-ubuntu", "clickhouse/test-old-centos"], ), run_command="compatibility_check.py", - runner_type=Runners.STYLE_CHECKER_ARM, + runner_type=Runners.STYLE_CHECKER, ) INSTALL_TEST = JobConfig( + job_name_keyword="install", digest=DigestConfig( include_paths=["./tests/ci/install_check.py"], docker=["clickhouse/install-deb-test", "clickhouse/install-rpm-test"], ), run_command='install_check.py "$CHECK_NAME"', - runner_type=Runners.STYLE_CHECKER_ARM, + runner_type=Runners.STYLE_CHECKER, timeout=900, ) STATELESS_TEST = JobConfig( + job_name_keyword="stateless", digest=DigestConfig( include_paths=[ "./tests/ci/functional_test_check.py", @@ -371,6 +377,7 @@ class CommonJobConfigs: timeout=10800, ) STATEFUL_TEST = JobConfig( + job_name_keyword="stateful", digest=DigestConfig( include_paths=[ "./tests/ci/functional_test_check.py", @@ -387,6 +394,7 @@ class CommonJobConfigs: timeout=3600, ) STRESS_TEST = JobConfig( + job_name_keyword="stress", digest=DigestConfig( include_paths=[ "./tests/queries/0_stateless/", @@ -403,6 +411,7 @@ class CommonJobConfigs: timeout=9000, ) UPGRADE_TEST = JobConfig( + job_name_keyword="upgrade", digest=DigestConfig( include_paths=["./tests/ci/upgrade_check.py"], exclude_files=[".md"], @@ -412,6 +421,7 @@ class CommonJobConfigs: runner_type=Runners.STRESS_TESTER, ) INTEGRATION_TEST = JobConfig( + job_name_keyword="integration", digest=DigestConfig( include_paths=[ "./tests/ci/integration_test_check.py", @@ -425,12 +435,14 @@ class CommonJobConfigs: runner_type=Runners.STRESS_TESTER, ) ASTFUZZER_TEST = JobConfig( + job_name_keyword="ast", digest=DigestConfig(), run_command="ast_fuzzer_check.py", run_always=True, runner_type=Runners.FUZZER_UNIT_TESTER, ) UNIT_TEST = JobConfig( + job_name_keyword="unit", digest=DigestConfig( include_paths=["./tests/ci/unit_tests_check.py"], exclude_files=[".md"], @@ -440,6 +452,7 @@ class CommonJobConfigs: runner_type=Runners.FUZZER_UNIT_TESTER, ) PERF_TESTS = JobConfig( + job_name_keyword="performance", digest=DigestConfig( include_paths=[ "./tests/ci/performance_comparison_check.py", @@ -452,6 +465,7 @@ class CommonJobConfigs: runner_type=Runners.STRESS_TESTER, ) SQLLANCER_TEST = JobConfig( + job_name_keyword="lancer", digest=DigestConfig(), run_command="sqlancer_check.py", release_only=True, @@ -459,6 +473,7 @@ class CommonJobConfigs: runner_type=Runners.FUZZER_UNIT_TESTER, ) SQLLOGIC_TEST = JobConfig( + job_name_keyword="logic", digest=DigestConfig( include_paths=["./tests/ci/sqllogic_test.py"], exclude_files=[".md"], @@ -467,9 +482,10 @@ class CommonJobConfigs: run_command="sqllogic_test.py", timeout=10800, release_only=True, - runner_type=Runners.STYLE_CHECKER_ARM, + runner_type=Runners.STYLE_CHECKER, ) SQL_TEST = JobConfig( + job_name_keyword="sqltest", digest=DigestConfig( include_paths=["./tests/ci/sqltest.py"], exclude_files=[".md"], @@ -481,12 +497,14 @@ class CommonJobConfigs: runner_type=Runners.FUZZER_UNIT_TESTER, ) BUGFIX_TEST = JobConfig( + job_name_keyword="bugfix", digest=DigestConfig(), run_command="bugfix_validate_check.py", timeout=900, runner_type=Runners.FUNC_TESTER, ) DOCKER_SERVER = JobConfig( + job_name_keyword="docker", required_on_release_branch=True, run_command='docker_server.py --check-name "$CHECK_NAME" --release-type head --allow-build-reuse', digest=DigestConfig( @@ -498,6 +516,7 @@ class CommonJobConfigs: runner_type=Runners.STYLE_CHECKER, ) CLICKBENCH_TEST = JobConfig( + job_name_keyword="clickbench", digest=DigestConfig( include_paths=[ "tests/ci/clickbench.py", diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py index d8208a5d9bc..7a51a65b5d5 100644 --- a/tests/ci/test_ci_config.py +++ b/tests/ci/test_ci_config.py @@ -20,6 +20,42 @@ class TestCIConfig(unittest.TestCase): """check runner is provided w/o exception""" for job in CI.JobNames: self.assertIn(CI.JOB_CONFIGS[job].runner_type, CI.Runners) + if ( + job + in ( + CI.JobNames.STYLE_CHECK, + CI.JobNames.BUILD_CHECK, + ) + or "jepsen" in job.lower() + ): + self.assertTrue( + "style" in CI.JOB_CONFIGS[job].runner_type, + f"Job [{job}] must have style-checker(-aarch64) runner", + ) + elif "binary_" in job.lower() or "package_" in job.lower(): + self.assertTrue( + CI.JOB_CONFIGS[job].runner_type == CI.Runners.BUILDER, + f"Job [{job}] must have [{CI.Runners.BUILDER}] runner", + ) + elif "aarch64" in job.lower(): + self.assertTrue( + "aarch" in CI.JOB_CONFIGS[job].runner_type, + f"Job [{job}] does not match runner [{CI.JOB_CONFIGS[job].runner_type}]", + ) + else: + self.assertTrue( + "aarch" not in CI.JOB_CONFIGS[job].runner_type, + f"Job [{job}] does not match runner [{CI.JOB_CONFIGS[job].runner_type}]", + ) + + def test_common_configs_applied_properly(self): + for job in CI.JobNames: + if CI.JOB_CONFIGS[job].job_name_keyword: + self.assertTrue( + CI.JOB_CONFIGS[job].job_name_keyword.lower() + in normalize_string(job), + f"Job [{job}] apparently uses wrong common config with job keyword [{CI.JOB_CONFIGS[job].job_name_keyword}]", + ) def test_required_checks(self): for job in CI.REQUIRED_CHECKS: diff --git a/tests/ci/test_ci_options.py b/tests/ci/test_ci_options.py index fc21c7dda4e..fac6662b1aa 100644 --- a/tests/ci/test_ci_options.py +++ b/tests/ci/test_ci_options.py @@ -170,14 +170,10 @@ class TestCIOptions(unittest.TestCase): job: CI.JobConfig(runner_type=CI.Runners.STYLE_CHECKER) for job in _TEST_JOB_LIST } - jobs_configs[ - "fuzzers" - ].run_by_label = ( + jobs_configs["fuzzers"].run_by_label = ( "TEST_LABEL" # check "fuzzers" appears in the result due to the label ) - jobs_configs[ - "Integration tests (asan)" - ].release_only = ( + jobs_configs["Integration tests (asan)"].release_only = ( True # still must be included as it's set with include keywords ) filtered_jobs = list( @@ -309,9 +305,9 @@ class TestCIOptions(unittest.TestCase): job: CI.JobConfig(runner_type=CI.Runners.STYLE_CHECKER) for job in _TEST_JOB_LIST } - jobs_configs[ - "fuzzers" - ].run_by_label = "TEST_LABEL" # check "fuzzers" does not appears in the result + jobs_configs["fuzzers"].run_by_label = ( + "TEST_LABEL" # check "fuzzers" does not appears in the result + ) jobs_configs["Integration tests (asan)"].release_only = True filtered_jobs = list( ci_options.apply(