thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Automatic backports of important fixes to cloud-release

Browse Source 
* Automatic backports to cloud-release

---------

Co-authored-by: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
This commit is contained in:
pufit 2023-05-23 06:47:18 -04:00 committed by Mikhail f. Shiryaev
parent 2ac600b642
commit 8df7a69147
No known key found for this signature in database
GPG Key ID: 4B02ED204C7D93F4
1 changed files with 99 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

132
tests/ci/cherry_pick.py
View File

@ -45,8 +45,10 @@ from ssh import SSHKey
class Labels:
MUST_BACKPORT = "pr-must-backport"
MUST_BACKPORT_CLOUD = "pr-must-backport-cloud"
BACKPORT = "pr-backport"
BACKPORTS_CREATED = "pr-backports-created"
BACKPORTS_CREATED_CLOUD = "pr-backports-created-cloud"
CHERRYPICK = "pr-cherrypick"
DO_NOT_TEST = "do not test"
@ -68,9 +70,9 @@ This pull-request will be merged automatically as it reaches the mergeable state
### If the PR was closed and then reopened
If it stuck, check #{pr_number} for `{label_backports_created}` and delete it if \
If it stuck, check {pr_url} for `{label_backports_created}` and delete it if \
necessary. Manually merging will do nothing, since `{label_backports_created}` \
prevents the original PR #{pr_number} from being processed.
prevents the original PR {pr_url} from being processed.
"""
BACKPORT_DESCRIPTION = """This pull-request is a last step of an automated \
backporting.
@ -80,14 +82,17 @@ close it.
"""
REMOTE = ""
def __init__(self, name: str, pr: PullRequest):
def __init__(self, name: str, pr: PullRequest, repo: Repository):
self.name = name
self.pr = pr
self.repo = repo
self.cherrypick_branch = f"cherrypick/{name}/{pr.merge_commit_sha}"
self.backport_branch = f"backport/{name}/{pr.number}"
self.cherrypick_pr = None # type: Optional[PullRequest]
self.backport_pr = None # type: Optional[PullRequest]
self._backported = None # type: Optional[bool]
self._backported = False
self.git_prefix = ( # All commits to cherrypick are done as robot-clickhouse
"git -c user.email=robot-clickhouse@users.noreply.github.com "
"-c user.name=robot-clickhouse -c commit.gpgsign=false"
@ -188,7 +193,7 @@ close it.
f"{self.cherrypick_branch} {self.pr.merge_commit_sha}"
)
# Check if there actually any changes between branches. If no, then no
# Check if there are actually any changes between branches. If no, then no
# other actions are required. It's possible when changes are backported
# manually to the release branch already
try:
@ -216,10 +221,11 @@ close it.
for branch in [self.cherrypick_branch, self.backport_branch]:
git_runner(f"{self.git_prefix} push -f {self.REMOTE} {branch}:{branch}")
self.cherrypick_pr = self.pr.base.repo.create_pull(
self.cherrypick_pr = self.repo.create_pull(
title=f"Cherry pick #{self.pr.number} to {self.name}: {self.pr.title}",
body=self.CHERRYPICK_DESCRIPTION.format(
pr_number=self.pr.number,
pr_url=self.pr.html_url,
label_backports_created=Labels.BACKPORTS_CREATED,
),
base=self.backport_branch,
@ -253,9 +259,9 @@ close it.
f"{self.git_prefix} push -f {self.REMOTE} "
f"{self.backport_branch}:{self.backport_branch}"
)
self.backport_pr = self.pr.base.repo.create_pull(
self.backport_pr = self.repo.create_pull(
title=title,
body=f"Original pull-request #{self.pr.number}\n"
body=f"Original pull-request {self.pr.url}\n"
f"Cherry-pick pull-request #{self.cherrypick_pr.number}\n\n"
f"{self.BACKPORT_DESCRIPTION}",
base=self.name,
@ -314,22 +320,33 @@ close it.
@property
def backported(self) -> bool:
if self._backported is not None:
return self._backported
return self.backport_pr is not None
return self._backported or self.backport_pr is not None
def __repr__(self):
return self.name
class Backport:
def __init__(self, gh: GitHub, repo: str, dry_run: bool):
def __init__(
self,
gh: GitHub,
repo: str,
fetch_from: Optional[str],
dry_run: bool,
must_create_backport_label: str,
backport_created_label: str,
):
self.gh = gh
self._repo_name = repo
self._fetch_from = fetch_from
self.dry_run = dry_run
self._query = f"type:pr repo:{repo}"
self.must_create_backport_label = must_create_backport_label
self.backport_created_label = backport_created_label
self._remote = ""
self._remote_line = ""
self._repo = None # type: Optional[Repository]
self.release_prs = [] # type: PullRequests
self.release_branches = [] # type: List[str]
@ -338,25 +355,38 @@ class Backport:
self.error = None # type: Optional[Exception]
@property
def remote(self) -> str:
if not self._remote:
def remote_line(self) -> str:
if not self._remote_line:
# lines of "origin git@github.com:ClickHouse/ClickHouse.git (fetch)"
remotes = git_runner("git remote -v").split("\n")
# We need the first word from the first matching result
self._remote = tuple(
remote.split(maxsplit=1)[0]
self._remote_line = next(
iter(
remote
for remote in remotes
if f"github.com/{self._repo_name}" in remote # https
or f"github.com:{self._repo_name}" in remote # ssh
)[0]
)
)
return self._remote_line
@property
def remote(self) -> str:
if not self._remote:
self._remote = self.remote_line.split(maxsplit=1)[0]
git_runner(f"git fetch {self._remote}")
ReleaseBranch.REMOTE = self._remote
return self._remote
@property
def is_remote_ssh(self) -> bool:
return "github.com:" in self.remote_line
def receive_release_prs(self):
logging.info("Getting release PRs")
self.release_prs = self.gh.get_pulls_from_search(
query=f"{self._query} is:open",
query=f"type:pr repo:{self._repo_name} is:open",
sort="created",
order="asc",
label="release",
@ -365,6 +395,14 @@ class Backport:
self.labels_to_backport = [
f"v{branch}-must-backport" for branch in self.release_branches
]
if self._fetch_from:
logging.info("Fetching from %s", self._fetch_from)
fetch_from_repo = self.gh.get_repo(self._fetch_from)
git_runner(
f"git fetch {fetch_from_repo.ssh_url if self.is_remote_ssh else fetch_from_repo.clone_url} {fetch_from_repo.default_branch} --no-tags"
)
logging.info("Active releases: %s", ", ".join(self.release_branches))
def update_local_release_branches(self):
@ -396,9 +434,10 @@ class Backport:
# To not have a possible TZ issues
tomorrow = date.today() + timedelta(days=1)
logging.info("Receive PRs suppose to be backported")
self.prs_for_backport = self.gh.get_pulls_from_search(
query=f"{self._query} -label:{Labels.BACKPORTS_CREATED}",
label=",".join(self.labels_to_backport + [Labels.MUST_BACKPORT]),
query=f"type:pr repo:{self._fetch_from} -label:{self.backport_created_label}",
label=",".join(self.labels_to_backport + [self.must_create_backport_label]),
merged=[since_date, tomorrow],
)
logging.info(
@ -418,13 +457,13 @@ class Backport:
def process_pr(self, pr: PullRequest) -> None:
pr_labels = [label.name for label in pr.labels]
if Labels.MUST_BACKPORT in pr_labels:
if self.must_create_backport_label in pr_labels:
branches = [
ReleaseBranch(br, pr) for br in self.release_branches
ReleaseBranch(br, pr, self.repo) for br in self.release_branches
] # type: List[ReleaseBranch]
else:
branches = [
ReleaseBranch(br, pr)
ReleaseBranch(br, pr, self.repo)
for br in [
label.split("-", 1)[0][1:] # v21.8-must-backport
for label in pr_labels
@ -452,14 +491,14 @@ class Backport:
]
)
bp_cp_prs = self.gh.get_pulls_from_search(
query=f"{self._query} {query_suffix}",
query=f"type:pr repo:{self._repo_name} {query_suffix}",
)
for br in branches:
br.pop_prs(bp_cp_prs)
if bp_cp_prs:
# This is definitely some error. All prs must be consumed by
# branches with ReleaseBranch.pop_prs. It also make the whole
# branches with ReleaseBranch.pop_prs. It also makes the whole
# program exit code non-zero
self.error = Exception(
"The following PRs are not filtered by release branches:\n"
@ -483,22 +522,17 @@ class Backport:
if self.dry_run:
logging.info("DRY RUN: would mark PR #%s as done", pr.number)
return
pr.add_to_labels(Labels.BACKPORTS_CREATED)
pr.add_to_labels(self.backport_created_label)
logging.info(
"PR #%s is successfully labeled with `%s`",
pr.number,
Labels.BACKPORTS_CREATED,
self.backport_created_label,
)
@property
def repo(self) -> Repository:
if self._repo is None:
try:
self._repo = self.release_prs[0].base.repo
except IndexError as exc:
raise Exception(
"`repo` is available only after the `receive_release_prs`"
) from exc
self._repo = self.gh.get_repo(self._repo_name)
return self._repo
@property
@ -512,7 +546,26 @@ def parse_args():
parser.add_argument(
"--repo", default="ClickHouse/ClickHouse", help="repo owner/name"
)
parser.add_argument(
"--from-repo",
help="if set, the commits will be taken from this repo, but PRs will be created in the main repo",
)
parser.add_argument("--dry-run", action="store_true", help="do not create anything")
parser.add_argument(
"--must-create-backport-label",
default=Labels.MUST_BACKPORT,
choices=(Labels.MUST_BACKPORT, Labels.MUST_BACKPORT_CLOUD),
help="label to filter PRs to backport",
)
parser.add_argument(
"--backport-created-label",
default=Labels.BACKPORTS_CREATED,
choices=(Labels.BACKPORTS_CREATED, Labels.BACKPORTS_CREATED_CLOUD),
help="label to mark PRs as backported",
)
parser.add_argument(
"--debug-helpers",
action="store_true",
@ -564,7 +617,14 @@ def main():
token = args.token or get_best_robot_token()
gh = GitHub(token, create_cache_dir=False)
bp = Backport(gh, args.repo, args.dry_run)
bp = Backport(
gh,
args.repo,
args.from_repo,
args.dry_run,
args.must_create_backport_label,
args.backport_created_label,
)
# https://github.com/python/mypy/issues/3004
bp.gh.cache_path = f"{TEMP_PATH}/gh_cache" # type: ignore
bp.receive_release_prs()
@ -577,7 +637,7 @@ def main():
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
logging.getLogger().setLevel(level=logging.INFO)
assert not is_shallow()
with stash():