diff --git a/ci/praktika/job.py b/ci/praktika/job.py index 595a86456e9..e0e0cf18c8b 100644 --- a/ci/praktika/job.py +++ b/ci/praktika/job.py @@ -131,3 +131,10 @@ class Job: def __repr__(self): return self.name + + def copy(self): + """ + To create an instant copy of a job config used in multiple workflows + :return: Job.Config + """ + return copy.deepcopy(self) diff --git a/ci/praktika/mangle.py b/ci/praktika/mangle.py index c090604d59b..25d41f3efbb 100644 --- a/ci/praktika/mangle.py +++ b/ci/praktika/mangle.py @@ -1,13 +1,16 @@ import copy import importlib.util from pathlib import Path +from typing import List + +from praktika import Workflow from . import Job from .settings import Settings from .utils import Utils -def _get_workflows(name=None, file=None): +def _get_workflows(name=None, file=None) -> List[Workflow.Config]: """ Gets user's workflow configs """ diff --git a/ci/praktika/parser.py b/ci/praktika/parser.py index cb895fd6271..b029ded5fb2 100644 --- a/ci/praktika/parser.py +++ b/ci/praktika/parser.py @@ -50,6 +50,7 @@ class WorkflowYaml: artifact_to_config: Dict[str, ArtifactYaml] secret_names_gh: List[str] enable_cache: bool + cron_schedules: List[str] class WorkflowConfigParser: @@ -75,6 +76,7 @@ class WorkflowConfigParser: job_to_config={}, artifact_to_config={}, enable_cache=False, + cron_schedules=config.cron_schedules, ) def parse(self): diff --git a/ci/praktika/validator.py b/ci/praktika/validator.py index 9c0155cf889..bc189c7e338 100644 --- a/ci/praktika/validator.py +++ b/ci/praktika/validator.py @@ -1,5 +1,4 @@ import glob -import sys from itertools import chain from pathlib import Path @@ -21,6 +20,33 @@ class Validator: cls.validate_requirements_txt_files(workflow) cls.validate_dockers(workflow) + if workflow.event == Workflow.Event.SCHEDULE: + cls.evaluate_check( + workflow.cron_schedules + and isinstance(workflow.cron_schedules, list), + f".crone_schedules str must be non-empty list of cron strings .event===SCHEDULE, provided value [{workflow.cron_schedules}]", + workflow.name, + ) + for cron_schedule in workflow.cron_schedules: + cls.evaluate_check( + len(cron_schedule.split(" ")) == 5, + f".crone_schedules must be posix compliant cron str, e.g. '30 15 * * *', provided value [{cron_schedule}]", + workflow.name, + ) + for cron_token in cron_schedule.split(" ")[:-1]: + cls.evaluate_check( + cron_token == "*" or str.isdigit(cron_token), + f".crone_schedules must be posix compliant cron str, e.g. '30 15 * * 1,3', provided value [{cron_schedule}], invalid part [{cron_token}]", + workflow.name, + ) + days_of_weak = cron_schedule.split(" ")[-1] + cls.evaluate_check( + days_of_weak == "*" + or any([str.isdigit(v) for v in days_of_weak.split(",")]), + f".crone_schedules must be posix compliant cron str, e.g. '30 15 * * 1,3', provided value [{cron_schedule}], invalid part [{days_of_weak}]", + workflow.name, + ) + if workflow.artifacts: for artifact in workflow.artifacts: if artifact.is_s3_artifact(): @@ -198,4 +224,4 @@ class Validator: ) for message in messages: print(" || " + message) - sys.exit(1) + raise diff --git a/ci/praktika/workflow.py b/ci/praktika/workflow.py index 93c0653bc99..5f50a071b37 100644 --- a/ci/praktika/workflow.py +++ b/ci/praktika/workflow.py @@ -11,6 +11,8 @@ class Workflow: class Event: PULL_REQUEST = "pull_request" PUSH = "push" + SCHEDULE = "schedule" + DISPATCH = "dispatch" @dataclass class Config: @@ -32,6 +34,7 @@ class Workflow: enable_merge_ready_status: bool = False enable_cidb: bool = False enable_merge_commit: bool = False + cron_schedules: List[str] = field(default_factory=list) def is_event_pull_request(self): return self.event == Workflow.Event.PULL_REQUEST @@ -39,6 +42,9 @@ class Workflow: def is_event_push(self): return self.event == Workflow.Event.PUSH + def is_event_schedule(self): + return self.event == Workflow.Event.SCHEDULE + def get_job(self, name): job = self.find_job(name) if not job: diff --git a/ci/praktika/yaml_generator.py b/ci/praktika/yaml_generator.py index 814b32f8e68..b9b75de1ebe 100644 --- a/ci/praktika/yaml_generator.py +++ b/ci/praktika/yaml_generator.py @@ -37,19 +37,13 @@ jobs: {JOBS}\ """ - TEMPLATE_CALLABLE_WORKFLOW = """\ + TEMPLATE_SCHEDULE = """\ # generated by praktika name: {NAME} on: - workflow_call: - inputs: - config: - type: string - required: false - default: '' - secrets: -{SECRETS} + schedule:{CRON_TEMPLATES} + workflow_dispatch: env: PYTHONUNBUFFERED: 1 @@ -58,6 +52,10 @@ jobs: {JOBS}\ """ + TEMPLATE_CRON = """ + - cron: {CRON_SCHEDULE}\ +""" + TEMPLATE_SECRET_CONFIG = """\ {SECRET_NAME}: required: true @@ -88,9 +86,6 @@ jobs: cat > {ENV_SETUP_SCRIPT} << 'ENV_SETUP_SCRIPT_EOF' export PYTHONPATH=./ci:. {SETUP_ENVS} - cat > {WORKFLOW_CONFIG_FILE} << 'EOF' - ${{{{ needs.{WORKFLOW_CONFIG_JOB_NAME}.outputs.data }}}} - EOF cat > {WORKFLOW_STATUS_FILE} << 'EOF' ${{{{ toJson(needs) }}}} EOF @@ -119,6 +114,12 @@ jobs: )\ """ + TEMPLATE_SETUP_ENV_WF_CONFIG = """\ + cat > {WORKFLOW_CONFIG_FILE} << 'EOF' + ${{{{ needs.{WORKFLOW_CONFIG_JOB_NAME}.outputs.data }}}} + EOF\ +""" + TEMPLATE_PY_INSTALL = """ - name: Set up Python uses: actions/setup-python@v5 @@ -183,6 +184,7 @@ jobs: if ( workflow_config.is_event_pull_request() or workflow_config.is_event_push() + or workflow_config.is_event_schedule() ): yaml_workflow_str = PullRequestPushYamlGen(parser).generate() else: @@ -264,10 +266,18 @@ class PullRequestPushYamlGen: SECRET_NAME=secret ) ) + if self.workflow_config.enable_cache: + secrets_envs.append( + YamlGenerator.Templates.TEMPLATE_SETUP_ENV_WF_CONFIG.format( + WORKFLOW_CONFIG_FILE=RunConfig.file_name_static( + self.workflow_config.name + ), + WORKFLOW_CONFIG_JOB_NAME=config_job_name_normalized, + ) + ) job_item = YamlGenerator.Templates.TEMPLATE_JOB_0.format( JOB_NAME_NORMALIZED=job_name_normalized, - WORKFLOW_CONFIG_JOB_NAME=config_job_name_normalized, IF_EXPRESSION=if_expression, RUNS_ON=", ".join(job.runs_on), NEEDS=needs, @@ -278,9 +288,6 @@ class PullRequestPushYamlGen: WORKFLOW_NAME=self.workflow_config.name, ENV_SETUP_SCRIPT=Settings.ENV_SETUP_SCRIPT, SETUP_ENVS="\n".join(secrets_envs), - WORKFLOW_CONFIG_FILE=RunConfig.file_name_static( - self.workflow_config.name - ), JOB_ADDONS="".join(job_addons), DOWNLOADS_GITHUB="\n".join(downloads_github), UPLOADS_GITHUB="\n".join(uploads_github), @@ -293,14 +300,33 @@ class PullRequestPushYamlGen: ) job_items.append(job_item) - base_template = YamlGenerator.Templates.TEMPLATE_PULL_REQUEST_0 + # for schedule workflows only + cron_items = "" + for cron_item in self.workflow_config.cron_schedules: + cron_items += YamlGenerator.Templates.TEMPLATE_CRON.format( + CRON_SCHEDULE=cron_item + ) + + if self.workflow_config.event in (Workflow.Event.PULL_REQUEST,): + base_template = YamlGenerator.Templates.TEMPLATE_PULL_REQUEST_0 + format_kwargs = { + "BRANCHES": ", ".join( + [f"'{branch}'" for branch in self.workflow_config.branches] + ), + "EVENT": self.workflow_config.event, + } + elif self.workflow_config.event in (Workflow.Event.SCHEDULE,): + base_template = YamlGenerator.Templates.TEMPLATE_SCHEDULE + format_kwargs = {"CRON_TEMPLATES": cron_items} + else: + assert ( + False + ), f"Invalid or Not implemented event [{self.workflow_config.event}]" + template_1 = base_template.strip().format( NAME=self.workflow_config.name, - BRANCHES=", ".join( - [f"'{branch}'" for branch in self.workflow_config.branches] - ), - EVENT=self.workflow_config.event, JOBS="{}" * len(job_items), + **format_kwargs, ) res = template_1.format(*job_items) diff --git a/ci/workflows/defs.py b/ci/workflows/defs.py index 077921dc5da..357629e2f23 100644 --- a/ci/workflows/defs.py +++ b/ci/workflows/defs.py @@ -375,7 +375,7 @@ ARTIFACTS = [ class Jobs: style_check_job = Job.Config( name=JobNames.STYLE_CHECK, - runs_on=[RunnerLabels.CI_SERVICES], + runs_on=[RunnerLabels.STYLE_CHECK_ARM], command="python3 ./ci/jobs/check_style.py", run_in_docker="clickhouse/style-test", ) diff --git a/ci/workflows/packages_repo_backup.py b/ci/workflows/packages_repo_backup.py new file mode 100644 index 00000000000..6601ddbe642 --- /dev/null +++ b/ci/workflows/packages_repo_backup.py @@ -0,0 +1,16 @@ +from praktika import Workflow + +from ci.workflows.defs import Jobs + +nightly_workflow = Workflow.Config( + name="PackagesRepoBakUp", + event=Workflow.Event.SCHEDULE, + jobs=[ + Jobs.style_check_job, + ], + cron_schedules=["13 3 * * *"], +) + +WORKFLOWS = [ + nightly_workflow, +] diff --git a/ci/workflows/pull_request.py b/ci/workflows/pull_request.py index 363fc98913b..e996d4bce37 100644 --- a/ci/workflows/pull_request.py +++ b/ci/workflows/pull_request.py @@ -9,7 +9,7 @@ workflow = Workflow.Config( event=Workflow.Event.PULL_REQUEST, base_branches=[BASE_BRANCH], jobs=[ - Jobs.style_check_job, + Jobs.style_check_job.copy(), Jobs.fast_test_job, *Jobs.build_jobs, *Jobs.stateless_tests_jobs, diff --git a/utils/check-style/check-workflows b/utils/check-style/check-workflows index fb41d5af461..0fc15c30243 100755 --- a/utils/check-style/check-workflows +++ b/utils/check-style/check-workflows @@ -9,7 +9,7 @@ GIT_ROOT=$(git rev-parse --show-cdup) GIT_ROOT=${GIT_ROOT:-../../} act --list --directory="$GIT_ROOT" 1>/dev/null 2>&1 || act --list --directory="$GIT_ROOT" 2>&1 -actionlint -ignore 'reusable workflow call.+' || : +actionlint -ignore 'section should not be empty' || : python3 check_reusable_workflows.py