thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-18 20:32:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support scheduled workflows in praktika

Browse Source
This commit is contained in:
Max Kainov 2024-12-16 16:53:56 +01:00
parent 2f86b953ea
commit 3861337114
10 changed files with 113 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
ci/praktika/job.py
View File

@ -131,3 +131,10 @@ class Job:
def __repr__(self):
return self.name
def copy(self):
"""
To create an instant copy of a job config used in multiple workflows
:return: Job.Config
"""
return copy.deepcopy(self)

5
ci/praktika/mangle.py
View File

@ -1,13 +1,16 @@
import copy
import importlib.util
from pathlib import Path
from typing import List
from praktika import Workflow
from . import Job
from .settings import Settings
from .utils import Utils
def _get_workflows(name=None, file=None):
def _get_workflows(name=None, file=None) -> List[Workflow.Config]:
"""
Gets user's workflow configs
"""

2
ci/praktika/parser.py
View File

@ -50,6 +50,7 @@ class WorkflowYaml:
artifact_to_config: Dict[str, ArtifactYaml]
secret_names_gh: List[str]
enable_cache: bool
cron_schedules: List[str]
class WorkflowConfigParser:
@ -75,6 +76,7 @@ class WorkflowConfigParser:
job_to_config={},
artifact_to_config={},
enable_cache=False,
cron_schedules=config.cron_schedules,
)
def parse(self):

30
ci/praktika/validator.py
View File

@ -1,5 +1,4 @@
import glob
import sys
from itertools import chain
from pathlib import Path
@ -21,6 +20,33 @@ class Validator:
cls.validate_requirements_txt_files(workflow)
cls.validate_dockers(workflow)
if workflow.event == Workflow.Event.SCHEDULE:
cls.evaluate_check(
workflow.cron_schedules
and isinstance(workflow.cron_schedules, list),
f".crone_schedules str must be non-empty list of cron strings .event===SCHEDULE, provided value [{workflow.cron_schedules}]",
workflow.name,
)
for cron_schedule in workflow.cron_schedules:
cls.evaluate_check(
len(cron_schedule.split(" ")) == 5,
f".crone_schedules must be posix compliant cron str, e.g. '30 15 * * *', provided value [{cron_schedule}]",
workflow.name,
)
for cron_token in cron_schedule.split(" ")[:-1]:
cls.evaluate_check(
cron_token == "*" or str.isdigit(cron_token),
f".crone_schedules must be posix compliant cron str, e.g. '30 15 * * 1,3', provided value [{cron_schedule}], invalid part [{cron_token}]",
workflow.name,
)
days_of_weak = cron_schedule.split(" ")[-1]
cls.evaluate_check(
days_of_weak == "*"
or any([str.isdigit(v) for v in days_of_weak.split(",")]),
f".crone_schedules must be posix compliant cron str, e.g. '30 15 * * 1,3', provided value [{cron_schedule}], invalid part [{days_of_weak}]",
workflow.name,
)
if workflow.artifacts:
for artifact in workflow.artifacts:
if artifact.is_s3_artifact():
@ -198,4 +224,4 @@ class Validator:
)
for message in messages:
print(" || " + message)
sys.exit(1)
raise

6
ci/praktika/workflow.py
View File

@ -11,6 +11,8 @@ class Workflow:
class Event:
PULL_REQUEST = "pull_request"
PUSH = "push"
SCHEDULE = "schedule"
DISPATCH = "dispatch"
@dataclass
class Config:
@ -32,6 +34,7 @@ class Workflow:
enable_merge_ready_status: bool = False
enable_cidb: bool = False
enable_merge_commit: bool = False
cron_schedules: List[str] = field(default_factory=list)
def is_event_pull_request(self):
return self.event == Workflow.Event.PULL_REQUEST
@ -39,6 +42,9 @@ class Workflow:
def is_event_push(self):
return self.event == Workflow.Event.PUSH
def is_event_schedule(self):
return self.event == Workflow.Event.SCHEDULE
def get_job(self, name):
job = self.find_job(name)
if not job:

68
ci/praktika/yaml_generator.py
View File

@ -37,19 +37,13 @@ jobs:
{JOBS}\
"""
TEMPLATE_CALLABLE_WORKFLOW = """\
TEMPLATE_SCHEDULE = """\
# generated by praktika
name: {NAME}
on:
workflow_call:
inputs:
config:
type: string
required: false
default: ''
secrets:
{SECRETS}
schedule:{CRON_TEMPLATES}
workflow_dispatch:
env:
PYTHONUNBUFFERED: 1
@ -58,6 +52,10 @@ jobs:
{JOBS}\
"""
TEMPLATE_CRON = """
- cron: {CRON_SCHEDULE}\
"""
TEMPLATE_SECRET_CONFIG = """\
{SECRET_NAME}:
required: true
@ -88,9 +86,6 @@ jobs:
cat > {ENV_SETUP_SCRIPT} << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
{SETUP_ENVS}
cat > {WORKFLOW_CONFIG_FILE} << 'EOF'
${{{{ needs.{WORKFLOW_CONFIG_JOB_NAME}.outputs.data }}}}
EOF
cat > {WORKFLOW_STATUS_FILE} << 'EOF'
${{{{ toJson(needs) }}}}
EOF
@ -119,6 +114,12 @@ jobs:
)\
"""
TEMPLATE_SETUP_ENV_WF_CONFIG = """\
cat > {WORKFLOW_CONFIG_FILE} << 'EOF'
${{{{ needs.{WORKFLOW_CONFIG_JOB_NAME}.outputs.data }}}}
EOF\
"""
TEMPLATE_PY_INSTALL = """
- name: Set up Python
uses: actions/setup-python@v5
@ -183,6 +184,7 @@ jobs:
if (
workflow_config.is_event_pull_request()
or workflow_config.is_event_push()
or workflow_config.is_event_schedule()
):
yaml_workflow_str = PullRequestPushYamlGen(parser).generate()
else:
@ -264,10 +266,18 @@ class PullRequestPushYamlGen:
SECRET_NAME=secret
)
)
if self.workflow_config.enable_cache:
secrets_envs.append(
YamlGenerator.Templates.TEMPLATE_SETUP_ENV_WF_CONFIG.format(
WORKFLOW_CONFIG_FILE=RunConfig.file_name_static(
self.workflow_config.name
),
WORKFLOW_CONFIG_JOB_NAME=config_job_name_normalized,
)
)
job_item = YamlGenerator.Templates.TEMPLATE_JOB_0.format(
JOB_NAME_NORMALIZED=job_name_normalized,
WORKFLOW_CONFIG_JOB_NAME=config_job_name_normalized,
IF_EXPRESSION=if_expression,
RUNS_ON=", ".join(job.runs_on),
NEEDS=needs,
@ -278,9 +288,6 @@ class PullRequestPushYamlGen:
WORKFLOW_NAME=self.workflow_config.name,
ENV_SETUP_SCRIPT=Settings.ENV_SETUP_SCRIPT,
SETUP_ENVS="\n".join(secrets_envs),
WORKFLOW_CONFIG_FILE=RunConfig.file_name_static(
self.workflow_config.name
),
JOB_ADDONS="".join(job_addons),
DOWNLOADS_GITHUB="\n".join(downloads_github),
UPLOADS_GITHUB="\n".join(uploads_github),
@ -293,14 +300,33 @@ class PullRequestPushYamlGen:
)
job_items.append(job_item)
base_template = YamlGenerator.Templates.TEMPLATE_PULL_REQUEST_0
# for schedule workflows only
cron_items = ""
for cron_item in self.workflow_config.cron_schedules:
cron_items += YamlGenerator.Templates.TEMPLATE_CRON.format(
CRON_SCHEDULE=cron_item
)
if self.workflow_config.event in (Workflow.Event.PULL_REQUEST,):
base_template = YamlGenerator.Templates.TEMPLATE_PULL_REQUEST_0
format_kwargs = {
"BRANCHES": ", ".join(
[f"'{branch}'" for branch in self.workflow_config.branches]
),
"EVENT": self.workflow_config.event,
}
elif self.workflow_config.event in (Workflow.Event.SCHEDULE,):
base_template = YamlGenerator.Templates.TEMPLATE_SCHEDULE
format_kwargs = {"CRON_TEMPLATES": cron_items}
else:
assert (
False
), f"Invalid or Not implemented event [{self.workflow_config.event}]"
template_1 = base_template.strip().format(
NAME=self.workflow_config.name,
BRANCHES=", ".join(
[f"'{branch}'" for branch in self.workflow_config.branches]
),
EVENT=self.workflow_config.event,
JOBS="{}" * len(job_items),
**format_kwargs,
)
res = template_1.format(*job_items)

2
ci/workflows/defs.py
View File

@ -375,7 +375,7 @@ ARTIFACTS = [
class Jobs:
style_check_job = Job.Config(
name=JobNames.STYLE_CHECK,
runs_on=[RunnerLabels.CI_SERVICES],
runs_on=[RunnerLabels.STYLE_CHECK_ARM],
command="python3 ./ci/jobs/check_style.py",
run_in_docker="clickhouse/style-test",
)

16
ci/workflows/packages_repo_backup.py Normal file
View File

@ -0,0 +1,16 @@
from praktika import Workflow
from ci.workflows.defs import Jobs
nightly_workflow = Workflow.Config(
name="PackagesRepoBakUp",
event=Workflow.Event.SCHEDULE,
jobs=[
Jobs.style_check_job,
],
cron_schedules=["13 3 * * *"],
)
WORKFLOWS = [
nightly_workflow,
]

2
ci/workflows/pull_request.py
View File

@ -9,7 +9,7 @@ workflow = Workflow.Config(
event=Workflow.Event.PULL_REQUEST,
base_branches=[BASE_BRANCH],
jobs=[
Jobs.style_check_job,
Jobs.style_check_job.copy(),
Jobs.fast_test_job,
*Jobs.build_jobs,
*Jobs.stateless_tests_jobs,

2
utils/check-style/check-workflows
View File

@ -9,7 +9,7 @@ GIT_ROOT=$(git rev-parse --show-cdup)
GIT_ROOT=${GIT_ROOT:-../../}
act --list --directory="$GIT_ROOT" 1>/dev/null 2>&1 || act --list --directory="$GIT_ROOT" 2>&1
actionlint -ignore 'reusable workflow call.+' || :
actionlint -ignore 'section should not be empty' || :
python3 check_reusable_workflows.py