ClickHouse/tests/ci/test_ci_options.py

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

338 lines
10 KiB
Python
Raw Normal View History

#!/usr/bin/env python3
# type: ignore
import unittest
2024-06-02 16:25:14 +00:00
from ci_settings import CiSettings
2024-06-10 09:18:03 +00:00
from ci_config import CI
_TEST_BODY_1 = """
#### Run only:
2024-05-31 10:01:16 +00:00
- [x] <!---ci_set_non_required--> Non required
- [ ] <!---ci_set_arm--> Integration tests (arm64)
- [x] <!---ci_include_foo--> Integration tests
- [x] <!---ci_include_foo_Bar--> Integration tests
- [ ] <!---ci_include_bar--> Integration tests
- [x] <!---ci_exclude_foo--> some invalid mask - should be skipped
- [x] <!---ci_exclude_Foo_bar--> Integration tests
- [ ] <!---ci_exclude_bar--> Integration tests
#### CI options:
- [ ] <!---do_not_test--> do not test (only style check)
- [x] <!---woolen_wolfdog--> Woolen Wolfdog CI
- [x] <!---no_merge_commit--> disable merge-commit (no merge from master before tests)
- [ ] <!---no_ci_cache--> disable CI cache (job reuse)
#### Only specified batches in multi-batch jobs:
- [x] <!---batch_0--> 1
- [ ] <!---batch_1--> 2
"""
_TEST_BODY_2 = """
- [x] <!---ci_include_integration--> MUST include integration tests
- [x] <!---ci_include_stateless--> MUST include stateless tests
2024-04-30 20:07:22 +00:00
- [x] <!---ci_include_azure--> MUST include azure
- [x] <!---ci_include_foo_Bar--> no action must be applied
- [ ] <!---ci_include_bar--> no action must be applied
2024-05-31 10:30:40 +00:00
- [x] <!---ci_exclude_tsan|foobar--> MUST exclude tsan
- [x] <!---ci_exclude_aarch64--> MUST exclude aarch64
- [x] <!---ci_exclude_analyzer--> MUST exclude test with analazer
- [ ] <!---ci_exclude_bar--> no action applied
- [x] <!---ci_exclude_s3_storage--> Must exclude statless test with s3 storage
- [x] <!---ci_exclude_coverage--> Must exclude tests on coverage build
"""
_TEST_BODY_3 = """
- [x] <!---ci_include_analyzer--> Must include all tests for analyzer
"""
2024-05-01 13:56:17 +00:00
_TEST_BODY_4 = """
"""
2024-04-11 15:15:47 +00:00
_TEST_JOB_LIST = [
"Style check",
"Fast test",
"package_release",
"package_asan",
2024-06-02 16:25:14 +00:00
"package_aarch64",
"package_release_coverage",
"package_debug",
"package_tsan",
"package_msan",
"package_ubsan",
"binary_release",
"fuzzers",
2024-04-11 15:15:47 +00:00
"Docker server image",
"Docker keeper image",
2024-06-15 15:38:57 +00:00
"Install packages (release)",
"Install packages (aarch64)",
2024-04-11 15:15:47 +00:00
"Stateless tests (debug)",
"Stateless tests (release)",
"Stateless tests (coverage)",
"Stateless tests (aarch64)",
"Stateless tests (asan)",
"Stateless tests (tsan)",
"Stateless tests (msan)",
"Stateless tests (ubsan)",
"Stateless tests (release, old analyzer, s3, DatabaseReplicated)",
"Stateless tests (debug, s3 storage)",
"Stateless tests (tsan, s3 storage)",
"Stateless tests flaky check (asan)",
2024-04-30 20:07:22 +00:00
"Stateless tests (azure, asan)",
2024-04-11 15:15:47 +00:00
"Stateful tests (debug)",
"Stateful tests (release)",
"Stateful tests (coverage)",
"Stateful tests (aarch64)",
"Stateful tests (asan)",
"Stateful tests (tsan)",
"Stateful tests (msan)",
"Stateful tests (ubsan)",
"Stateful tests (release, ParallelReplicas)",
"Stateful tests (debug, ParallelReplicas)",
"Stateful tests (asan, ParallelReplicas)",
"Stateful tests (msan, ParallelReplicas)",
"Stateful tests (ubsan, ParallelReplicas)",
"Stateful tests (tsan, ParallelReplicas)",
"Stress test (asan)",
"Stress test (tsan)",
"Stress test (ubsan)",
"Stress test (msan)",
"Stress test (debug)",
"Integration tests (release)",
"Integration tests (asan)",
"Integration tests (asan, old analyzer)",
"Integration tests (tsan)",
"Integration tests (aarch64)",
"Integration tests flaky check (asan)",
"Upgrade check (debug)",
"Upgrade check (asan)",
"Upgrade check (tsan)",
"Upgrade check (msan)",
"Unit tests (release)",
"Unit tests (asan)",
"Unit tests (msan)",
"Unit tests (tsan)",
"Unit tests (ubsan)",
"AST fuzzer (debug)",
"AST fuzzer (asan)",
"AST fuzzer (msan)",
"AST fuzzer (tsan)",
"AST fuzzer (ubsan)",
"ClickHouse Keeper Jepsen",
"ClickHouse Server Jepsen",
2024-06-15 15:38:57 +00:00
"Performance Comparison (release)",
"Performance Comparison (aarch64)",
2024-04-11 15:15:47 +00:00
"Sqllogic test (release)",
"SQLancer (release)",
"SQLancer (debug)",
"SQLTest",
2024-06-15 15:38:57 +00:00
"Compatibility check (release)",
2024-04-11 15:15:47 +00:00
"Compatibility check (aarch64)",
2024-06-15 15:38:57 +00:00
"ClickBench (release)",
2024-04-11 15:15:47 +00:00
"ClickBench (aarch64)",
"libFuzzer tests",
"Builds",
2024-04-11 15:15:47 +00:00
"Docs check",
"Bugfix validation",
]
2024-06-02 16:25:14 +00:00
_TEST_JOB_LIST_2 = ["Style check", "Fast test", "fuzzers"]
class TestCIOptions(unittest.TestCase):
def test_pr_body_parsing(self):
2024-06-02 16:25:14 +00:00
ci_options = CiSettings.create_from_pr_message(
_TEST_BODY_1, update_from_api=False
)
self.assertFalse(ci_options.do_not_test)
self.assertFalse(ci_options.no_ci_cache)
self.assertTrue(ci_options.no_merge_commit)
self.assertTrue(ci_options.woolen_wolfdog)
2024-05-31 10:01:16 +00:00
self.assertEqual(ci_options.ci_sets, ["ci_set_non_required"])
self.assertCountEqual(ci_options.include_keywords, ["foo", "foo_bar"])
self.assertCountEqual(ci_options.exclude_keywords, ["foo", "foo_bar"])
def test_options_applied(self):
self.maxDiff = None
2024-06-02 16:25:14 +00:00
ci_options = CiSettings.create_from_pr_message(
_TEST_BODY_2, update_from_api=False
)
self.assertFalse(ci_options.woolen_wolfdog)
self.assertCountEqual(
2024-04-30 20:17:32 +00:00
ci_options.include_keywords,
["integration", "foo_bar", "stateless", "azure"],
)
self.assertCountEqual(
ci_options.exclude_keywords,
2024-05-31 10:30:40 +00:00
["tsan", "foobar", "aarch64", "analyzer", "s3_storage", "coverage"],
)
2024-06-02 16:25:14 +00:00
2024-06-10 09:18:03 +00:00
jobs_configs = {
job: CI.JobConfig(runner_type=CI.Runners.STYLE_CHECKER)
for job in _TEST_JOB_LIST
}
2024-06-22 11:51:20 +00:00
jobs_configs[
"fuzzers"
].run_by_label = (
2024-06-02 16:25:14 +00:00
"TEST_LABEL" # check "fuzzers" appears in the result due to the label
)
2024-06-22 11:51:20 +00:00
jobs_configs[
"Integration tests (asan)"
].release_only = (
2024-06-02 16:25:14 +00:00
True # still must be included as it's set with include keywords
)
filtered_jobs = list(
ci_options.apply(
2024-06-06 23:35:39 +00:00
jobs_configs,
is_release=False,
is_pr=True,
is_mq=False,
labels=["TEST_LABEL"],
2024-06-02 16:25:14 +00:00
)
)
self.assertCountEqual(
2024-06-02 16:25:14 +00:00
filtered_jobs,
[
"Style check",
2024-06-02 16:25:14 +00:00
"fuzzers",
"package_release",
"package_asan",
2024-06-02 16:25:14 +00:00
"package_debug",
"package_msan",
"package_ubsan",
"Stateless tests (asan)",
2024-04-30 20:07:22 +00:00
"Stateless tests (azure, asan)",
"Stateless tests flaky check (asan)",
"Stateless tests (msan)",
"Stateless tests (ubsan)",
"Stateless tests (debug)",
"Stateless tests (release)",
"Integration tests (release)",
"Integration tests (asan)",
"Integration tests flaky check (asan)",
],
)
def test_options_applied_2(self):
2024-06-10 09:18:03 +00:00
jobs_configs = {
job: CI.JobConfig(runner_type=CI.Runners.STYLE_CHECKER)
for job in _TEST_JOB_LIST_2
}
2024-06-02 16:25:14 +00:00
jobs_configs["Style check"].release_only = True
jobs_configs["Fast test"].pr_only = True
jobs_configs["fuzzers"].run_by_label = "TEST_LABEL"
# no settings are set
filtered_jobs = list(
2024-06-06 23:35:39 +00:00
CiSettings().apply(
jobs_configs, is_release=False, is_pr=False, is_mq=True, labels=[]
)
2024-06-02 16:25:14 +00:00
)
self.assertCountEqual(
filtered_jobs,
[
"Fast test",
],
)
filtered_jobs = list(
2024-06-06 23:35:39 +00:00
CiSettings().apply(
jobs_configs, is_release=False, is_pr=True, is_mq=False, labels=[]
)
)
self.assertCountEqual(
filtered_jobs,
[
"Fast test",
],
)
filtered_jobs = list(
CiSettings().apply(
jobs_configs, is_release=True, is_pr=False, is_mq=False, labels=[]
)
2024-06-02 16:25:14 +00:00
)
self.assertCountEqual(
filtered_jobs,
[
"Style check",
],
)
def test_options_applied_3(self):
ci_settings = CiSettings()
ci_settings.include_keywords = ["Style"]
2024-06-10 09:18:03 +00:00
jobs_configs = {
job: CI.JobConfig(runner_type=CI.Runners.STYLE_CHECKER)
for job in _TEST_JOB_LIST_2
}
2024-06-02 16:25:14 +00:00
jobs_configs["Style check"].release_only = True
jobs_configs["Fast test"].pr_only = True
# no settings are set
filtered_jobs = list(
ci_settings.apply(
2024-06-06 23:35:39 +00:00
jobs_configs,
is_release=False,
is_pr=True,
is_mq=False,
labels=["TEST_LABEL"],
2024-06-02 16:25:14 +00:00
)
)
self.assertCountEqual(
filtered_jobs,
[
"Style check",
],
)
ci_settings.include_keywords = ["Fast"]
filtered_jobs = list(
ci_settings.apply(
2024-06-06 23:35:39 +00:00
jobs_configs,
is_release=True,
is_pr=False,
is_mq=False,
labels=["TEST_LABEL"],
2024-06-02 16:25:14 +00:00
)
)
self.assertCountEqual(
filtered_jobs,
[
"Style check",
],
)
def test_options_applied_4(self):
self.maxDiff = None
2024-06-02 16:25:14 +00:00
ci_options = CiSettings.create_from_pr_message(
_TEST_BODY_3, update_from_api=False
)
self.assertCountEqual(ci_options.include_keywords, ["analyzer"])
self.assertIsNone(ci_options.exclude_keywords)
2024-06-10 09:18:03 +00:00
jobs_configs = {
job: CI.JobConfig(runner_type=CI.Runners.STYLE_CHECKER)
for job in _TEST_JOB_LIST
}
2024-06-22 11:51:20 +00:00
jobs_configs[
"fuzzers"
].run_by_label = "TEST_LABEL" # check "fuzzers" does not appears in the result
2024-06-02 16:25:14 +00:00
jobs_configs["Integration tests (asan)"].release_only = True
filtered_jobs = list(
ci_options.apply(
2024-06-06 23:35:39 +00:00
jobs_configs,
is_release=False,
is_pr=True,
is_mq=False,
labels=["TEST_LABEL"],
2024-06-02 16:25:14 +00:00
)
)
self.assertCountEqual(
2024-06-02 16:25:14 +00:00
filtered_jobs,
[
"Style check",
2024-03-27 10:39:33 +00:00
"Integration tests (asan, old analyzer)",
"package_release",
2024-03-27 10:39:33 +00:00
"Stateless tests (release, old analyzer, s3, DatabaseReplicated)",
"package_asan",
2024-06-02 16:25:14 +00:00
"fuzzers",
],
)