import dataclasses from typing import Any, Dict, List from . import Artifact, Workflow from .mangle import _get_workflows class AddonType: PY = "py" @dataclasses.dataclass class WorkflowYaml: @dataclasses.dataclass class JobYaml: name: str needs: List[str] runs_on: List[str] artifacts_gh_requires: List["WorkflowYaml.ArtifactYaml"] artifacts_gh_provides: List["WorkflowYaml.ArtifactYaml"] addons: List["WorkflowYaml.JobAddonYaml"] gh_app_auth: bool run_unless_cancelled: bool parameter: Any def __repr__(self): return self.name @dataclasses.dataclass class ArtifactYaml: name: str provided_by: str required_by: List[str] path: str type: str def __repr__(self): return self.name @dataclasses.dataclass class JobAddonYaml: install_python: bool requirements_txt_path: str name: str event: str branches: List[str] jobs: List[JobYaml] job_to_config: Dict[str, JobYaml] artifact_to_config: Dict[str, ArtifactYaml] secret_names_gh: List[str] enable_cache: bool class WorkflowConfigParser: def __init__(self, config: Workflow.Config): self.workflow_name = config.name self.config = config self.requires_all = [] # type: List[str] self.provides_all = [] # type: List[str] self.job_names_all = [] # type: List[str] self.artifact_to_providing_job_map = {} # type: Dict[str, List[str]] self.artifact_to_job_requires_map = {} # type: Dict[str, List[str]] self.artifact_map = {} # type: Dict[str, List[Artifact.Config]] self.job_to_provides_artifacts = {} # type: Dict[str, List[Artifact.Config]] self.job_to_requires_artifacts = {} # type: Dict[str, List[Artifact.Config]] self.workflow_yaml_config = WorkflowYaml( name=self.workflow_name, event=config.event, branches=[], jobs=[], secret_names_gh=[], job_to_config={}, artifact_to_config={}, enable_cache=False, ) def parse(self): self.workflow_yaml_config.enable_cache = self.config.enable_cache # populate WorkflowYaml.branches if self.config.event in (Workflow.Event.PUSH,): assert ( self.config.branches ), f'Workflow.Config.branches (e.g. ["main"]) must be set for workflow with event [{self.config.event}], workflow [{self.workflow_name}]' assert ( not self.config.base_branches ), f'Workflow.Config.base_branches (e.g. ["main"]) must not be set for workflow with event [{self.config.event}], workflow [{self.workflow_name}]' assert isinstance( self.config.branches, list ), f'Workflow.Config.branches must be of type list (e.g. ["main"]), workflow [{self.workflow_name}]' self.workflow_yaml_config.branches = self.config.branches elif self.config.event in (Workflow.Event.PULL_REQUEST,): assert ( self.config.base_branches ), f'Workflow.Config.base_branches (e.g. ["main"]) must be set for workflow with event [{self.config.event}], workflow [{self.workflow_name}]' assert ( not self.config.branches ), f'Workflow.Config.branches (e.g. ["main"]) must not be set for workflow with event [{self.config.event}], workflow [{self.workflow_name}]' assert isinstance( self.config.base_branches, list ), f'Workflow.Config.base_branches must be of type list (e.g. ["main"]), workflow [{self.workflow_name}]' self.workflow_yaml_config.branches = self.config.base_branches # populate WorkflowYaml.artifact_to_config with phony artifacts for job in self.config.jobs: assert ( job.name not in self.workflow_yaml_config.artifact_to_config ), f"Not uniq Job name [{job.name}], workflow [{self.workflow_name}]" self.workflow_yaml_config.artifact_to_config[job.name] = ( WorkflowYaml.ArtifactYaml( name=job.name, provided_by=job.name, required_by=[], path="", type=Artifact.Type.PHONY, ) ) # populate jobs for job in self.config.jobs: job_yaml_config = WorkflowYaml.JobYaml( name=job.name, addons=[], artifacts_gh_requires=[], artifacts_gh_provides=[], needs=[], runs_on=[], gh_app_auth=False, run_unless_cancelled=job.run_unless_cancelled, parameter=None, ) self.workflow_yaml_config.jobs.append(job_yaml_config) assert ( job.name not in self.workflow_yaml_config.job_to_config ), f"Job name [{job.name}] is not uniq, workflow [{self.workflow_name}]" self.workflow_yaml_config.job_to_config[job.name] = job_yaml_config # populate WorkflowYaml.artifact_to_config if self.config.artifacts: for artifact in self.config.artifacts: assert ( artifact.name not in self.workflow_yaml_config.artifact_to_config ), f"Artifact name [{artifact.name}] is not uniq, workflow [{self.workflow_name}]" artifact_yaml_config = WorkflowYaml.ArtifactYaml( name=artifact.name, provided_by="", required_by=[], path=artifact.path, type=artifact.type, ) self.workflow_yaml_config.artifact_to_config[artifact.name] = ( artifact_yaml_config ) # populate ArtifactYaml.provided_by for job in self.config.jobs: if job.provides: for artifact_name in job.provides: assert ( artifact_name in self.workflow_yaml_config.artifact_to_config ), f"Artifact [{artifact_name}] has no config, job [{job.name}], workflow [{self.workflow_name}]" assert not self.workflow_yaml_config.artifact_to_config[ artifact_name ].provided_by, f"Artifact [{artifact_name}] provided by multiple jobs [{self.workflow_yaml_config.artifact_to_config[artifact_name].provided_by}] and [{job.name}]" self.workflow_yaml_config.artifact_to_config[ artifact_name ].provided_by = job.name # populate ArtifactYaml.required_by for job in self.config.jobs: if job.requires: for artifact_name in job.requires: assert ( artifact_name in self.workflow_yaml_config.artifact_to_config ), f"Artifact [{artifact_name}] has no config, job [{job.name}], workflow [{self.workflow_name}]" assert self.workflow_yaml_config.artifact_to_config[ artifact_name ].provided_by, f"Artifact [{artifact_name}] has no job providing it, required by job [{job.name}], workflow [{self.workflow_name}]" self.workflow_yaml_config.artifact_to_config[ artifact_name ].required_by.append(job.name) # populate JobYaml.addons for job in self.config.jobs: if job.job_requirements: addon_yaml = WorkflowYaml.JobAddonYaml( requirements_txt_path=job.job_requirements.python_requirements_txt, install_python=job.job_requirements.python, ) self.workflow_yaml_config.job_to_config[job.name].addons.append( addon_yaml ) if self.config.enable_report: for job in self.config.jobs: # auth required for every job with enabled HTML, so that workflow summary status can be updated self.workflow_yaml_config.job_to_config[job.name].gh_app_auth = True # populate JobYaml.runs_on for job in self.config.jobs: self.workflow_yaml_config.job_to_config[job.name].runs_on = job.runs_on # populate JobYaml.artifacts_gh_requires, JobYaml.artifacts_gh_provides and JobYaml.needs for ( artifact_name, artifact, ) in self.workflow_yaml_config.artifact_to_config.items(): # assert ( # artifact.provided_by # and artifact.provided_by in self.workflow_yaml_config.job_to_config # ), f"Artifact [{artifact_name}] has no valid job providing it [{artifact.provided_by}]" for job_name in artifact.required_by: if ( artifact.provided_by not in self.workflow_yaml_config.job_to_config[job_name].needs ): self.workflow_yaml_config.job_to_config[job_name].needs.append( artifact.provided_by ) if artifact.type in (Artifact.Type.GH,): self.workflow_yaml_config.job_to_config[ job_name ].artifacts_gh_requires.append(artifact) elif artifact.type in (Artifact.Type.PHONY, Artifact.Type.S3): pass else: assert ( False ), f"Artifact [{artifact_name}] has unsupported type [{artifact.type}]" if not artifact.required_by and artifact.type != Artifact.Type.PHONY: print( f"WARNING: Artifact [{artifact_name}] provided by job [{artifact.provided_by}] not required by any job in workflow [{self.workflow_name}]" ) if artifact.type == Artifact.Type.GH: self.workflow_yaml_config.job_to_config[ artifact.provided_by ].artifacts_gh_provides.append(artifact) # populate JobYaml.parametrize for job in self.config.jobs: self.workflow_yaml_config.job_to_config[job.name].parameter = job.parameter # populate secrets for secret_config in self.config.secrets: if secret_config.is_gh(): self.workflow_yaml_config.secret_names_gh.append(secret_config.name) return self if __name__ == "__main__": # test workflows = _get_workflows() for workflow in workflows: WorkflowConfigParser(workflow).parse()