Support scheduled workflows in praktika

ci/praktika/job.py

@@ -131,3 +131,10 @@ class Job:
 
         def __repr__(self):
             return self.name
+
+        def copy(self):
+            """
+            To create an instant copy of a job config used in multiple workflows
+            :return: Job.Config
+            """
+            return copy.deepcopy(self)

ci/praktika/mangle.py

@@ -1,13 +1,16 @@
 import copy
 import importlib.util
 from pathlib import Path
+from typing import List
+
+from praktika import Workflow
 
 from . import Job
 from .settings import Settings
 from .utils import Utils
 
 
-def _get_workflows(name=None, file=None):
+def _get_workflows(name=None, file=None) -> List[Workflow.Config]:
     """
     Gets user's workflow configs
     """

ci/praktika/parser.py

@@ -50,6 +50,7 @@ class WorkflowYaml:
     artifact_to_config: Dict[str, ArtifactYaml]
     secret_names_gh: List[str]
     enable_cache: bool
+    cron_schedules: List[str]
 
 
 class WorkflowConfigParser:
@@ -75,6 +76,7 @@ class WorkflowConfigParser:
             job_to_config={},
             artifact_to_config={},
             enable_cache=False,
+            cron_schedules=config.cron_schedules,
         )
 
     def parse(self):

ci/praktika/validator.py

@@ -1,5 +1,4 @@
 import glob
-import sys
 from itertools import chain
 from pathlib import Path
 
@@ -21,6 +20,33 @@ class Validator:
             cls.validate_requirements_txt_files(workflow)
             cls.validate_dockers(workflow)
 
+            if workflow.event == Workflow.Event.SCHEDULE:
+                cls.evaluate_check(
+                    workflow.cron_schedules
+                    and isinstance(workflow.cron_schedules, list),
+                    f".crone_schedules str must be non-empty list of cron strings .event===SCHEDULE, provided value [{workflow.cron_schedules}]",
+                    workflow.name,
+                )
+                for cron_schedule in workflow.cron_schedules:
+                    cls.evaluate_check(
+                        len(cron_schedule.split(" ")) == 5,
+                        f".crone_schedules must be posix compliant cron str, e.g. '30 15 * * *', provided value [{cron_schedule}]",
+                        workflow.name,
+                    )
+                    for cron_token in cron_schedule.split(" ")[:-1]:
+                        cls.evaluate_check(
+                            cron_token == "*" or str.isdigit(cron_token),
+                            f".crone_schedules must be posix compliant cron str, e.g. '30 15 * * 1,3', provided value [{cron_schedule}], invalid part [{cron_token}]",
+                            workflow.name,
+                        )
+                    days_of_weak = cron_schedule.split(" ")[-1]
+                    cls.evaluate_check(
+                        days_of_weak == "*"
+                        or any([str.isdigit(v) for v in days_of_weak.split(",")]),
+                        f".crone_schedules must be posix compliant cron str, e.g. '30 15 * * 1,3', provided value [{cron_schedule}], invalid part [{days_of_weak}]",
+                        workflow.name,
+                    )
+
             if workflow.artifacts:
                 for artifact in workflow.artifacts:
                     if artifact.is_s3_artifact():
@@ -198,4 +224,4 @@ class Validator:
             )
             for message in messages:
                 print(" ||  " + message)
-            sys.exit(1)
+            raise

ci/praktika/workflow.py

@@ -11,6 +11,8 @@ class Workflow:
     class Event:
         PULL_REQUEST = "pull_request"
         PUSH = "push"
+        SCHEDULE = "schedule"
+        DISPATCH = "dispatch"
 
     @dataclass
     class Config:
@@ -32,6 +34,7 @@ class Workflow:
         enable_merge_ready_status: bool = False
         enable_cidb: bool = False
         enable_merge_commit: bool = False
+        cron_schedules: List[str] = field(default_factory=list)
 
         def is_event_pull_request(self):
             return self.event == Workflow.Event.PULL_REQUEST
@@ -39,6 +42,9 @@ class Workflow:
         def is_event_push(self):
             return self.event == Workflow.Event.PUSH
 
+        def is_event_schedule(self):
+            return self.event == Workflow.Event.SCHEDULE
+
         def get_job(self, name):
             job = self.find_job(name)
             if not job:

ci/praktika/yaml_generator.py

@@ -37,19 +37,13 @@ jobs:
 {JOBS}\
 """
 
-        TEMPLATE_CALLABLE_WORKFLOW = """\
+        TEMPLATE_SCHEDULE = """\
 # generated by praktika
 
 name: {NAME}
 on:
-  workflow_call:
+  schedule:{CRON_TEMPLATES}
-    inputs:
+  workflow_dispatch:
-      config:
-        type: string
-        required: false
-        default: ''
-    secrets:
-{SECRETS}
 
 env:
   PYTHONUNBUFFERED: 1
@@ -58,6 +52,10 @@ jobs:
 {JOBS}\
 """
 
+        TEMPLATE_CRON = """
+    - cron: {CRON_SCHEDULE}\
+"""
+
         TEMPLATE_SECRET_CONFIG = """\
       {SECRET_NAME}:
         required: true
@@ -88,9 +86,6 @@ jobs:
           cat > {ENV_SETUP_SCRIPT} << 'ENV_SETUP_SCRIPT_EOF'
           export PYTHONPATH=./ci:.
 {SETUP_ENVS}
-          cat > {WORKFLOW_CONFIG_FILE} << 'EOF'
-          ${{{{ needs.{WORKFLOW_CONFIG_JOB_NAME}.outputs.data }}}}
-          EOF
           cat > {WORKFLOW_STATUS_FILE} << 'EOF'
           ${{{{ toJson(needs) }}}}
           EOF
@@ -119,6 +114,12 @@ jobs:
           )\
 """
 
+        TEMPLATE_SETUP_ENV_WF_CONFIG = """\
+          cat > {WORKFLOW_CONFIG_FILE} << 'EOF'
+          ${{{{ needs.{WORKFLOW_CONFIG_JOB_NAME}.outputs.data }}}}
+          EOF\
+"""
+
         TEMPLATE_PY_INSTALL = """
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -183,6 +184,7 @@ jobs:
             if (
                 workflow_config.is_event_pull_request()
                 or workflow_config.is_event_push()
+                or workflow_config.is_event_schedule()
             ):
                 yaml_workflow_str = PullRequestPushYamlGen(parser).generate()
             else:
@@ -264,10 +266,18 @@ class PullRequestPushYamlGen:
                         SECRET_NAME=secret
                     )
                 )
+            if self.workflow_config.enable_cache:
+                secrets_envs.append(
+                    YamlGenerator.Templates.TEMPLATE_SETUP_ENV_WF_CONFIG.format(
+                        WORKFLOW_CONFIG_FILE=RunConfig.file_name_static(
+                            self.workflow_config.name
+                        ),
+                        WORKFLOW_CONFIG_JOB_NAME=config_job_name_normalized,
+                    )
+                )
 
             job_item = YamlGenerator.Templates.TEMPLATE_JOB_0.format(
                 JOB_NAME_NORMALIZED=job_name_normalized,
-                WORKFLOW_CONFIG_JOB_NAME=config_job_name_normalized,
                 IF_EXPRESSION=if_expression,
                 RUNS_ON=", ".join(job.runs_on),
                 NEEDS=needs,
@@ -278,9 +288,6 @@ class PullRequestPushYamlGen:
                 WORKFLOW_NAME=self.workflow_config.name,
                 ENV_SETUP_SCRIPT=Settings.ENV_SETUP_SCRIPT,
                 SETUP_ENVS="\n".join(secrets_envs),
-                WORKFLOW_CONFIG_FILE=RunConfig.file_name_static(
-                    self.workflow_config.name
-                ),
                 JOB_ADDONS="".join(job_addons),
                 DOWNLOADS_GITHUB="\n".join(downloads_github),
                 UPLOADS_GITHUB="\n".join(uploads_github),
@@ -293,14 +300,33 @@ class PullRequestPushYamlGen:
             )
             job_items.append(job_item)
 
-        base_template = YamlGenerator.Templates.TEMPLATE_PULL_REQUEST_0
+        # for schedule workflows only
+        cron_items = ""
+        for cron_item in self.workflow_config.cron_schedules:
+            cron_items += YamlGenerator.Templates.TEMPLATE_CRON.format(
+                CRON_SCHEDULE=cron_item
+            )
+
+        if self.workflow_config.event in (Workflow.Event.PULL_REQUEST,):
+            base_template = YamlGenerator.Templates.TEMPLATE_PULL_REQUEST_0
+            format_kwargs = {
+                "BRANCHES": ", ".join(
+                    [f"'{branch}'" for branch in self.workflow_config.branches]
+                ),
+                "EVENT": self.workflow_config.event,
+            }
+        elif self.workflow_config.event in (Workflow.Event.SCHEDULE,):
+            base_template = YamlGenerator.Templates.TEMPLATE_SCHEDULE
+            format_kwargs = {"CRON_TEMPLATES": cron_items}
+        else:
+            assert (
+                False
+            ), f"Invalid or Not implemented event [{self.workflow_config.event}]"
+
         template_1 = base_template.strip().format(
             NAME=self.workflow_config.name,
-            BRANCHES=", ".join(
-                [f"'{branch}'" for branch in self.workflow_config.branches]
-            ),
-            EVENT=self.workflow_config.event,
             JOBS="{}" * len(job_items),
+            **format_kwargs,
         )
         res = template_1.format(*job_items)
 

ci/workflows/defs.py

@@ -375,7 +375,7 @@ ARTIFACTS = [
 class Jobs:
     style_check_job = Job.Config(
         name=JobNames.STYLE_CHECK,
-        runs_on=[RunnerLabels.CI_SERVICES],
+        runs_on=[RunnerLabels.STYLE_CHECK_ARM],
         command="python3 ./ci/jobs/check_style.py",
         run_in_docker="clickhouse/style-test",
     )

ci/workflows/packages_repo_backup.py

@@ -0,0 +1,16 @@
+from praktika import Workflow
+
+from ci.workflows.defs import Jobs
+
+nightly_workflow = Workflow.Config(
+    name="PackagesRepoBakUp",
+    event=Workflow.Event.SCHEDULE,
+    jobs=[
+        Jobs.style_check_job,
+    ],
+    cron_schedules=["13 3 * * *"],
+)
+
+WORKFLOWS = [
+    nightly_workflow,
+]

ci/workflows/pull_request.py

@@ -9,7 +9,7 @@ workflow = Workflow.Config(
     event=Workflow.Event.PULL_REQUEST,
     base_branches=[BASE_BRANCH],
     jobs=[
-        Jobs.style_check_job,
+        Jobs.style_check_job.copy(),
         Jobs.fast_test_job,
         *Jobs.build_jobs,
         *Jobs.stateless_tests_jobs,

utils/check-style/check-workflows

@@ -9,7 +9,7 @@ GIT_ROOT=$(git rev-parse --show-cdup)
 GIT_ROOT=${GIT_ROOT:-../../}
 act --list --directory="$GIT_ROOT" 1>/dev/null 2>&1 || act --list --directory="$GIT_ROOT" 2>&1
 
-actionlint -ignore 'reusable workflow call.+' || :
+actionlint -ignore 'section should not be empty' || :
 
 
 python3 check_reusable_workflows.py