Merge pull request #65385 from ClickHouse/ci_non_blocking_mode

CI: Add Non-blocking (Woolen wolfdog) CI mode

.github/PULL_REQUEST_TEMPLATE.md

@@ -49,7 +49,6 @@ At a minimum, the following information should be added (but add more as needed)
 - [ ] <!---ci_include_integration--> Allow: Integration Tests
 - [ ] <!---ci_include_performance--> Allow: Performance tests
 - [ ] <!---ci_set_builds--> Allow: All Builds
-- [ ] <!---ci_set_non_required--> Allow: All NOT Required Checks
 - [ ] <!---batch_0_1--> Allow: batch 1, 2 for multi-batch jobs
 - [ ] <!---batch_2_3--> Allow: batch 3, 4, 5, 6 for multi-batch jobs
 ---
@@ -60,6 +59,7 @@ At a minimum, the following information should be added (but add more as needed)
 - [ ] <!---ci_exclude_aarch64|release|debug--> Exclude: All with aarch64, release, debug
 ---
 - [ ] <!---do_not_test--> Do not test
+- [ ] <!---woolen_wolfdog--> Woolen Wolfdog
 - [ ] <!---upload_all--> Upload binaries for special builds
 - [ ] <!---no_merge_commit--> Disable merge-commit
 - [ ] <!---no_ci_cache--> Disable CI cache

.github/workflows/pull_request.yml

@@ -126,8 +126,9 @@ jobs:
     with:
       stage: Builds_2
       data: ${{ needs.RunConfig.outputs.data }}
+  # stage for running non-required checks without being blocked by required checks (Test_1) if corresponding settings is selected
   Tests_2:
-    needs: [RunConfig, Builds_2]
+    needs: [RunConfig, Builds_1]
     if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
     uses: ./.github/workflows/reusable_test_stage.yml
     with:

tests/ci/ci.py

@@ -462,7 +462,9 @@ def _configure_jobs(
     return ci_cache
 
 
-def _generate_ci_stage_config(jobs_data: Dict[str, Any]) -> Dict[str, Dict[str, Any]]:
+def _generate_ci_stage_config(
+    jobs_data: Dict[str, Any], non_blocking_mode: bool = False
+) -> Dict[str, Dict[str, Any]]:
     """
     populates GH Actions' workflow with real jobs
     "Builds_1": [{"job_name": NAME, "runner_type": RUNNER_TYPE}]
@@ -472,7 +474,7 @@ def _generate_ci_stage_config(jobs_data: Dict[str, Any]) -> Dict[str, Dict[str,
     result = {}  # type: Dict[str, Any]
     stages_to_do = []
     for job in jobs_data:
-        stage_type = CI.get_job_ci_stage(job)
+        stage_type = CI.get_job_ci_stage(job, non_blocking_ci=non_blocking_mode)
         if stage_type == CI.WorkflowStages.NA:
             continue
         if stage_type not in result:
@@ -1007,7 +1009,9 @@ def main() -> int:
         result["docs"] = ci_cache.job_digests[CI.JobNames.DOCS_CHECK]
         result["ci_settings"] = ci_settings.as_dict()
         if not args.skip_jobs:
-            result["stages_data"] = _generate_ci_stage_config(ci_cache.jobs_to_do)
+            result["stages_data"] = _generate_ci_stage_config(
+                ci_cache.jobs_to_do, ci_settings.woolen_wolfdog
+            )
             result["jobs_data"] = {
                 "jobs_to_do": list(ci_cache.jobs_to_do),
                 "jobs_to_skip": ci_cache.jobs_to_skip,

tests/ci/ci_config.py

@@ -545,7 +545,7 @@ class CI:
         return None
 
     @classmethod
-    def get_job_ci_stage(cls, job_name: str) -> str:
+    def get_job_ci_stage(cls, job_name: str, non_blocking_ci: bool = False) -> str:
         if job_name in [
             JobNames.STYLE_CHECK,
             JobNames.FAST_TEST,
@@ -572,6 +572,8 @@ class CI:
                 else:
                     stage_type = WorkflowStages.TESTS_3
         assert stage_type, f"BUG [{job_name}]"
+        if non_blocking_ci and stage_type == WorkflowStages.TESTS_3:
+            stage_type = WorkflowStages.TESTS_2
         return stage_type
 
     @classmethod

tests/ci/ci_definitions.py

@@ -46,6 +46,7 @@ class Tags(metaclass=WithIter):
     """
 
     DO_NOT_TEST_LABEL = "do_not_test"
+    WOOLEN_WOLFDOG_LABEL = "woolen_wolfdog"
     NO_MERGE_COMMIT = "no_merge_commit"
     NO_CI_CACHE = "no_ci_cache"
     # to upload all binaries from build jobs

tests/ci/ci_settings.py

@@ -29,6 +29,7 @@ class CiSettings:
     no_ci_cache: bool = False
     upload_all: bool = False
     no_merge_commit: bool = False
+    woolen_wolfdog: bool = False
 
     def as_dict(self) -> Dict[str, Any]:
         return asdict(self)
@@ -108,6 +109,9 @@ class CiSettings:
             elif match == CI.Tags.NO_MERGE_COMMIT:
                 res.no_merge_commit = True
                 print("NOTE: Merge Commit will be disabled")
+            elif match == CI.Tags.WOOLEN_WOLFDOG_LABEL:
+                res.woolen_wolfdog = True
+                print("NOTE: Woolen Wolfdog mode enabled")
             elif match.startswith("batch_"):
                 batches = []
                 try:

tests/ci/test_ci_config.py

@@ -201,6 +201,37 @@ class TestCIConfig(unittest.TestCase):
                         msg=f"Stage for [{job}] is not correct",
                     )
 
+    def test_job_stage_config_non_blocking(self):
+        """
+        check runner is provided w/o exception
+        """
+        # check stages
+        for job in CI.JobNames:
+            if job in CI.BuildNames:
+                self.assertTrue(
+                    CI.get_job_ci_stage(job)
+                    in (CI.WorkflowStages.BUILDS_1, CI.WorkflowStages.BUILDS_2)
+                )
+            else:
+                if job in (
+                    CI.JobNames.STYLE_CHECK,
+                    CI.JobNames.FAST_TEST,
+                    CI.JobNames.JEPSEN_SERVER,
+                    CI.JobNames.JEPSEN_KEEPER,
+                    CI.JobNames.BUILD_CHECK,
+                ):
+                    self.assertEqual(
+                        CI.get_job_ci_stage(job),
+                        CI.WorkflowStages.NA,
+                        msg=f"Stage for [{job}] is not correct",
+                    )
+                else:
+                    self.assertTrue(
+                        CI.get_job_ci_stage(job, non_blocking_ci=True)
+                        in (CI.WorkflowStages.TESTS_1, CI.WorkflowStages.TESTS_2),
+                        msg=f"Stage for [{job}] is not correct",
+                    )
+
     def test_build_jobs_configs(self):
         """
         check build jobs have non-None build_config attribute

tests/ci/test_ci_options.py

@@ -19,6 +19,7 @@ _TEST_BODY_1 = """
 
 #### CI options:
 - [ ] <!---do_not_test--> do not test (only style check)
+- [x] <!---woolen_wolfdog--> Woolen Wolfdog CI
 - [x] <!---no_merge_commit--> disable merge-commit (no merge from master before tests)
 - [ ] <!---no_ci_cache--> disable CI cache (job reuse)
 
@@ -148,6 +149,7 @@ class TestCIOptions(unittest.TestCase):
         self.assertFalse(ci_options.do_not_test)
         self.assertFalse(ci_options.no_ci_cache)
         self.assertTrue(ci_options.no_merge_commit)
+        self.assertTrue(ci_options.woolen_wolfdog)
         self.assertEqual(ci_options.ci_sets, ["ci_set_non_required"])
         self.assertCountEqual(ci_options.include_keywords, ["foo", "foo_bar"])
         self.assertCountEqual(ci_options.exclude_keywords, ["foo", "foo_bar"])
@@ -157,6 +159,7 @@ class TestCIOptions(unittest.TestCase):
         ci_options = CiSettings.create_from_pr_message(
             _TEST_BODY_2, update_from_api=False
         )
+        self.assertFalse(ci_options.woolen_wolfdog)
         self.assertCountEqual(
             ci_options.include_keywords,
             ["integration", "foo_bar", "stateless", "azure"],