ClickHouse/tests/ci/cherry_pick.py
2024-11-11 16:12:01 +01:00

678 lines
25 KiB
Python

#!/usr/bin/env python3
"""
A plan:
- TODO: consider receiving GH objects cache from S3, but it's really a few
of requests to API currently
- Get all open release PRs (20.10, 21.8, 22.5, etc.)
- Get all pull-requests between the date of the merge-base for the oldest PR with
labels pr-must-backport and version-specific v21.8-must-backport, but without
pr-backported
- Iterate over gotten PRs:
- for pr-must-backport:
- check if all backport-PRs are created. If yes,
set pr-backported label and finish
- If not, create either cherrypick PRs or merge cherrypick (in the same
stage, if mergable) and create backport-PRs
- If successfull, set pr-backported label on the PR
- for version-specific labels:
- the same, check, cherry-pick, backport, pr-backported
Cherry-pick stage:
- From time to time the cherry-pick fails, if it was done manually. In the
case we check if it's even needed, and mark the release as done somehow.
"""
import argparse
import logging
import os
from contextlib import contextmanager
from datetime import date, datetime, timedelta
from pathlib import Path
from subprocess import CalledProcessError
from typing import List, Optional
import __main__
from ci_buddy import CIBuddy
from ci_config import Labels
from env_helper import IS_CI, TEMP_PATH
from get_robot_token import get_best_robot_token
from git_helper import GIT_PREFIX, git_runner, is_shallow
from github_helper import GitHub, PullRequest, PullRequests, Repository
from ssh import SSHKey
class ReleaseBranch:
CHERRYPICK_DESCRIPTION = """Original pull-request #{pr_number}
This pull-request is a first step of an automated backporting.
It contains changes similar to calling `git cherry-pick` locally.
If you intend to continue backporting the changes, then resolve all conflicts if any.
Otherwise, if you do not want to backport them, then just close this pull-request.
The check results does not matter at this step - you can safely ignore them.
### Note
This pull-request will be merged automatically. Please, **do not merge it manually** \
(but if you accidentally did, nothing bad will happen).
### Troubleshooting
#### If the PR was manually reopened after being closed
If this PR is stuck (i.e. not automatically merged after one day), check {pr_url} for \
`{backport_created_label}` *label* and delete it.
Manually merging will do nothing. The `{backport_created_label}` *label* prevents the \
original PR {pr_url} from being processed.
#### If the conflicts were resolved in a wrong way
If this cherry-pick PR is completely screwed by a wrong conflicts resolution, and you \
want to recreate it:
- delete the `{label_cherrypick}` label from the PR
- delete this branch from the repository
You also need to check the original PR {pr_url} for `{backport_created_label}`, and \
delete if it's presented there
"""
BACKPORT_DESCRIPTION = """This pull-request is a last step of an automated \
backporting.
Treat it as a standard pull-request: look at the checks and resolve conflicts.
Merge it only if you intend to backport changes to the target branch, otherwise just \
close it.
"""
REMOTE = ""
def __init__(
self,
name: str,
pr: PullRequest,
repo: Repository,
backport_created_label: str,
):
self.name = name
self.pr = pr
self.repo = repo
self.cherrypick_branch = f"cherrypick/{name}/{pr.number}"
self.backport_branch = f"backport/{name}/{pr.number}"
self.cherrypick_pr = None # type: Optional[PullRequest]
self.backport_pr = None # type: Optional[PullRequest]
self._backported = False
self.backport_created_label = backport_created_label
self.pre_check()
def pre_check(self):
branch_updated = git_runner(
f"git branch -a --contains={self.pr.merge_commit_sha} "
f"{self.REMOTE}/{self.name}"
)
if branch_updated:
self._backported = True
def pop_prs(self, prs: PullRequests) -> PullRequests:
"""the method processes all prs and pops the ReleaseBranch related prs"""
to_pop = [] # type: List[int]
for i, pr in enumerate(prs):
if self.name not in pr.head.ref:
# this pr is not for the current branch
continue
if pr.head.ref.startswith(f"cherrypick/{self.name}"):
self.cherrypick_pr = pr
to_pop.append(i)
elif pr.head.ref.startswith(f"backport/{self.name}"):
self.backport_pr = pr
self._backported = True
to_pop.append(i)
else:
assert False, f"BUG! Invalid PR's branch [{pr.head.ref}]"
for i in reversed(to_pop):
# Going from the tail to keep the order and pop greater index first
prs.pop(i)
return prs
def process( # pylint: disable=too-many-return-statements
self, dry_run: bool
) -> None:
if self.backported:
return
if not self.cherrypick_pr:
if dry_run:
logging.info(
"DRY RUN: Would create cherrypick PR for #%s", self.pr.number
)
return
self.create_cherrypick()
assert self.cherrypick_pr, "BUG!"
if self.cherrypick_pr.mergeable and self.cherrypick_pr.state != "closed":
if dry_run:
logging.info(
"DRY RUN: Would merge cherry-pick PR for #%s", self.pr.number
)
return
self.cherrypick_pr.merge()
# The PR needs update, since PR.merge doesn't update the object
self.cherrypick_pr.update()
if self.cherrypick_pr.merged:
if dry_run:
logging.info(
"DRY RUN: Would create backport PR for #%s", self.pr.number
)
return
self.create_backport()
return
if self.cherrypick_pr.state == "closed":
logging.info(
"The cherry-pick PR #%s for PR #%s is discarded",
self.cherrypick_pr.number,
self.pr.number,
)
self._backported = True
return
logging.info(
"Cherry-pick PR #%s for PR #%s has conflicts and unable to be merged",
self.cherrypick_pr.number,
self.pr.number,
)
self.ping_cherry_pick_assignees(dry_run)
def create_cherrypick(self):
# First, create backport branch:
# Checkout release branch with discarding every change
git_runner(f"{GIT_PREFIX} checkout -f {self.name}")
# Create or reset backport branch
git_runner(f"{GIT_PREFIX} checkout -B {self.backport_branch}")
# Merge all changes from PR's the first parent commit w/o applying anything
# It will allow to create a merge commit like it would be a cherry-pick
first_parent = git_runner(f"git rev-parse {self.pr.merge_commit_sha}^1")
git_runner(f"{GIT_PREFIX} merge -s ours --no-edit {first_parent}")
# Second step, create cherrypick branch
git_runner(
f"{GIT_PREFIX} branch -f "
f"{self.cherrypick_branch} {self.pr.merge_commit_sha}"
)
# Check if there are actually any changes between branches. If no, then no
# other actions are required. It's possible when changes are backported
# manually to the release branch already
try:
output = git_runner(
f"{GIT_PREFIX} merge --no-commit --no-ff {self.cherrypick_branch}"
)
# 'up-to-date', 'up to date', who knows what else (╯°v°)╯ ^┻━┻
if output.startswith("Already up") and output.endswith("date."):
# The changes are already in the release branch, we are done here
logging.info(
"Release branch %s already contain changes from %s",
self.name,
self.pr.number,
)
self._backported = True
return
except CalledProcessError:
# There are most probably conflicts, they'll be resolved in PR
git_runner(f"{GIT_PREFIX} reset --merge")
else:
# There are changes to apply, so continue
git_runner(f"{GIT_PREFIX} reset --merge")
# Push, create the cherry-pick PR, label and assign it
for branch in [self.cherrypick_branch, self.backport_branch]:
git_runner(f"{GIT_PREFIX} push -f {self.REMOTE} {branch}:{branch}")
self.cherrypick_pr = self.repo.create_pull(
title=f"Cherry pick #{self.pr.number} to {self.name}: {self.pr.title}",
body=self.CHERRYPICK_DESCRIPTION.format(
pr_number=self.pr.number,
pr_url=self.pr.html_url,
backport_created_label=self.backport_created_label,
label_cherrypick=Labels.PR_CHERRYPICK,
),
base=self.backport_branch,
head=self.cherrypick_branch,
)
self.cherrypick_pr.add_to_labels(Labels.PR_CHERRYPICK)
self.cherrypick_pr.add_to_labels(Labels.DO_NOT_TEST)
if Labels.PR_CRITICAL_BUGFIX in [label.name for label in self.pr.labels]:
self.cherrypick_pr.add_to_labels(Labels.PR_CRITICAL_BUGFIX)
elif Labels.PR_BUGFIX in [label.name for label in self.pr.labels]:
self.cherrypick_pr.add_to_labels(Labels.PR_BUGFIX)
self._assign_new_pr(self.cherrypick_pr)
# update cherrypick PR to get the state for PR.mergable
self.cherrypick_pr.update()
def create_backport(self):
assert self.cherrypick_pr is not None
# Checkout the backport branch from the remote and make all changes to
# apply like they are only one cherry-pick commit on top of release
logging.info("Creating backport for PR #%s", self.pr.number)
git_runner(f"{GIT_PREFIX} checkout -f {self.backport_branch}")
git_runner(f"{GIT_PREFIX} pull --ff-only {self.REMOTE} {self.backport_branch}")
merge_base = git_runner(
f"{GIT_PREFIX} merge-base "
f"{self.REMOTE}/{self.name} {self.backport_branch}"
)
git_runner(f"{GIT_PREFIX} reset --soft {merge_base}")
title = f"Backport #{self.pr.number} to {self.name}: {self.pr.title}"
git_runner(f"{GIT_PREFIX} commit --allow-empty -F -", input=title)
# Push with force, create the backport PR, lable and assign it
git_runner(
f"{GIT_PREFIX} push -f {self.REMOTE} "
f"{self.backport_branch}:{self.backport_branch}"
)
self.backport_pr = self.repo.create_pull(
title=title,
body=f"Original pull-request {self.pr.html_url}\n"
f"Cherry-pick pull-request #{self.cherrypick_pr.number}\n\n"
f"{self.BACKPORT_DESCRIPTION}",
base=self.name,
head=self.backport_branch,
)
self.backport_pr.add_to_labels(Labels.PR_BACKPORT)
if Labels.PR_CRITICAL_BUGFIX in [label.name for label in self.pr.labels]:
self.backport_pr.add_to_labels(Labels.PR_CRITICAL_BUGFIX)
elif Labels.PR_BUGFIX in [label.name for label in self.pr.labels]:
self.backport_pr.add_to_labels(Labels.PR_BUGFIX)
self._assign_new_pr(self.backport_pr)
def ping_cherry_pick_assignees(self, dry_run: bool) -> None:
assert self.cherrypick_pr is not None
logging.info(
"Checking if cherry-pick PR #%s needs to be pinged",
self.cherrypick_pr.number,
)
# The `updated_at` is Optional[datetime]
cherrypick_updated_ts = (
self.cherrypick_pr.updated_at or datetime.now()
).timestamp()
since_updated = int(datetime.now().timestamp() - cherrypick_updated_ts)
since_updated_str = (
f"{since_updated // 86400}d{since_updated // 3600}"
f"h{since_updated // 60 % 60}m{since_updated % 60}s"
)
if since_updated < 86400:
logging.info(
"The cherry-pick PR was updated %s ago, "
"waiting for the next running",
since_updated_str,
)
return
assignees = ", ".join(f"@{user.login}" for user in self.cherrypick_pr.assignees)
comment_body = (
f"Dear {assignees}, the PR is not updated for {since_updated_str}. "
"Please, either resolve the conflicts, or close it to finish "
f"the backport process of #{self.pr.number}"
)
if dry_run:
logging.info(
"DRY RUN: would comment the cherry-pick PR #%s:\n",
self.cherrypick_pr.number,
)
return
self.cherrypick_pr.create_issue_comment(comment_body)
def _assign_new_pr(self, new_pr: PullRequest) -> None:
"""Assign `new_pr` to author, merger and assignees of an original PR"""
# It looks there some race when multiple .add_to_assignees are executed,
# so we'll add all at once
assignees = [self.pr.user, self.pr.merged_by]
if self.pr.assignees:
assignees.extend(self.pr.assignees)
logging.info(
"Assing #%s to author and assignees of the original PR: %s",
new_pr.number,
", ".join(user.login for user in assignees),
)
new_pr.add_to_assignees(*assignees)
@property
def backported(self) -> bool:
return self._backported
def __repr__(self):
return self.name
class Backport:
def __init__(
self,
gh: GitHub,
repo: str,
fetch_from: Optional[str],
dry_run: bool,
):
self.gh = gh
self._repo_name = repo
self._fetch_from = fetch_from
self.dry_run = dry_run
self.must_create_backport_labels = (
[Labels.MUST_BACKPORT]
if self._repo_name == self._fetch_from
else [Labels.MUST_BACKPORT_CLOUD, Labels.MUST_BACKPORT]
)
self.backport_created_label = (
Labels.PR_BACKPORTS_CREATED
if self._repo_name == self._fetch_from
else Labels.PR_BACKPORTS_CREATED_CLOUD
)
self._remote = ""
self._remote_line = ""
self._repo = None # type: Optional[Repository]
self.release_prs = [] # type: PullRequests
self.release_branches = [] # type: List[str]
self.labels_to_backport = [] # type: List[str]
self.prs_for_backport = [] # type: PullRequests
self.error = None # type: Optional[Exception]
@property
def remote_line(self) -> str:
if not self._remote_line:
# lines of "origin git@github.com:ClickHouse/ClickHouse.git (fetch)"
remotes = git_runner("git remote -v").split("\n")
# We need the first word from the first matching result
self._remote_line = next(
iter(
remote
for remote in remotes
if f"github.com/{self._repo_name}" in remote # https
or f"github.com:{self._repo_name}" in remote # ssh
)
)
return self._remote_line
@property
def remote(self) -> str:
if not self._remote:
self._remote = self.remote_line.split(maxsplit=1)[0]
git_runner(f"git fetch {self._remote}")
ReleaseBranch.REMOTE = self._remote
return self._remote
@property
def is_remote_ssh(self) -> bool:
return "github.com:" in self.remote_line
def receive_release_prs(self):
logging.info("Getting release PRs")
self.release_prs = self.gh.get_release_pulls(self._repo_name)
self.release_branches = [pr.head.ref for pr in self.release_prs]
if not self._fetch_from:
self.labels_to_backport = [
f"v{branch}-must-backport" for branch in self.release_branches
]
else:
self.labels_to_backport = [
(
f"v{branch}-must-backport"
if self._repo_name == "ClickHouse/ClickHouse"
else f"v{branch.replace('release/','')}-must-backport"
)
for branch in self.release_branches
]
logging.info("Fetching from %s", self._fetch_from)
fetch_from_repo = self.gh.get_repo(self._fetch_from)
git_runner(
"git fetch "
f"{fetch_from_repo.ssh_url if self.is_remote_ssh else fetch_from_repo.clone_url} "
f"{fetch_from_repo.default_branch} --no-tags"
)
logging.info("Active releases: %s", ", ".join(self.release_branches))
def update_local_release_branches(self):
logging.info("Update local release branches")
branches = git_runner("git branch").split()
for branch in self.release_branches:
if branch not in branches:
# the local branch is not exist, so continue
continue
local_ref = git_runner(f"git rev-parse {branch}")
remote_ref = git_runner(f"git rev-parse {self.remote}/{branch}")
if local_ref == remote_ref:
# Do not need to update, continue
continue
logging.info("Resetting %s to %s/%s", branch, self.remote, branch)
git_runner(f"git branch -f {branch} {self.remote}/{branch}")
def receive_prs_for_backport(self, reserve_search_days: int) -> None:
# The commits in the oldest open release branch
oldest_branch_commits = git_runner(
"git log --no-merges --format=%H --reverse "
f"{self.remote}/{self.default_branch}..{self.remote}/{self.release_branches[0]}"
)
# The first commit is the one we are looking for
since_commit = oldest_branch_commits.split("\n", 1)[0]
since_date = date.fromisoformat(
git_runner.run(f"git log -1 --format=format:%cs {since_commit}")
) - timedelta(days=reserve_search_days)
# To not have a possible TZ issues
tomorrow = date.today() + timedelta(days=1)
logging.info("Receive PRs supposed to be backported")
query_args = {
"query": f"type:pr repo:{self._fetch_from} -label:{self.backport_created_label}",
"label": ",".join(
self.labels_to_backport + self.must_create_backport_labels
),
"merged": [since_date, tomorrow],
}
logging.info("Query to find the backport PRs:\n %s", query_args)
self.prs_for_backport = self.gh.get_pulls_from_search(**query_args)
logging.info(
"PRs to be backported:\n %s",
"\n ".join([pr.html_url for pr in self.prs_for_backport]),
)
def process_backports(self):
for pr in self.prs_for_backport:
try:
self.process_pr(pr)
except Exception as e:
logging.error(
"During processing the PR #%s error occurred: %s", pr.number, e
)
self.error = e
def process_pr(self, pr: PullRequest) -> None:
pr_labels = [label.name for label in pr.labels]
if any(label in pr_labels for label in self.must_create_backport_labels):
branches = [
ReleaseBranch(br, pr, self.repo, self.backport_created_label)
for br in self.release_branches
] # type: List[ReleaseBranch]
else:
branches = [
ReleaseBranch(
(
br
if self._repo_name == "ClickHouse/ClickHouse"
else f"release/{br}"
),
pr,
self.repo,
self.backport_created_label,
)
for br in [
label.split("-", 1)[0][1:] # v21.8-must-backport
for label in pr_labels
if label in self.labels_to_backport
]
]
assert branches, "BUG!"
logging.info(
" PR #%s is supposed to be backported to %s",
pr.number,
", ".join(map(str, branches)),
)
# All PRs for cherry-pick and backport branches as heads
query_suffix = " ".join(
[
f"head:{branch.backport_branch} head:{branch.cherrypick_branch}"
for branch in branches
]
)
bp_cp_prs = self.gh.get_pulls_from_search(
query=f"type:pr repo:{self._repo_name} {query_suffix}",
label=f"{Labels.PR_BACKPORT},{Labels.PR_CHERRYPICK}",
)
for br in branches:
bp_cp_prs = br.pop_prs(bp_cp_prs)
assert not bp_cp_prs, "BUG!"
for br in branches:
br.process(self.dry_run)
if all(br.backported for br in branches):
# And check it after the running
self.mark_pr_backported(pr)
def mark_pr_backported(self, pr: PullRequest) -> None:
if self.dry_run:
logging.info("DRY RUN: would mark PR #%s as done", pr.number)
return
pr.add_to_labels(self.backport_created_label)
logging.info(
"PR #%s is successfully labeled with `%s`",
pr.number,
self.backport_created_label,
)
@property
def repo(self) -> Repository:
if self._repo is None:
self._repo = self.gh.get_repo(self._repo_name)
return self._repo
@property
def default_branch(self) -> str:
return self.repo.default_branch
def parse_args():
parser = argparse.ArgumentParser("Create cherry-pick and backport PRs")
parser.add_argument("--token", help="github token, if not set, used from smm")
parser.add_argument(
"--repo", default="ClickHouse/ClickHouse", help="repo owner/name"
)
parser.add_argument(
"--from-repo",
default="ClickHouse/ClickHouse",
help="if set, the commits will be taken from this repo, but PRs will be created in the main repo",
)
parser.add_argument("--dry-run", action="store_true", help="do not create anything")
parser.add_argument(
"--reserve-search-days",
default=0,
type=int,
help="safity reserve for the PRs search days, necessary for cloud",
)
parser.add_argument(
"--debug-helpers",
action="store_true",
help="add debug logging for git_helper and github_helper",
)
return parser.parse_args()
@contextmanager
def clear_repo():
def ref():
return git_runner("git branch --show-current") or git_runner(
"git rev-parse HEAD"
)
orig_ref = ref()
try:
yield
finally:
current_ref = ref()
if orig_ref != current_ref:
git_runner(f"git checkout -f {orig_ref}")
@contextmanager
def stash():
# diff.ignoreSubmodules=all don't show changed submodules
need_stash = bool(git_runner("git -c diff.ignoreSubmodules=all diff HEAD"))
if need_stash:
script = (
__main__.__file__ if hasattr(__main__, "__file__") else "unknown script"
)
git_runner(f"git stash push --no-keep-index -m 'running {script}'")
try:
with clear_repo():
yield
finally:
if need_stash:
git_runner("git stash pop")
def main():
temp_path = Path(TEMP_PATH)
temp_path.mkdir(parents=True, exist_ok=True)
args = parse_args()
if args.debug_helpers:
logging.getLogger("github_helper").setLevel(logging.DEBUG)
logging.getLogger("git_helper").setLevel(logging.DEBUG)
token = args.token or get_best_robot_token()
gh = GitHub(token, create_cache_dir=False)
bp = Backport(
gh,
args.repo,
args.from_repo,
args.dry_run,
)
# https://github.com/python/mypy/issues/3004
bp.gh.cache_path = temp_path / "gh_cache"
bp.receive_release_prs()
bp.update_local_release_branches()
bp.receive_prs_for_backport(args.reserve_search_days)
bp.process_backports()
if bp.error is not None:
logging.error("Finished successfully, but errors occurred!")
if IS_CI:
ci_buddy = CIBuddy()
ci_buddy.post_job_error(
f"The cherry-pick finished with errors: {bp.error}",
with_instance_info=True,
with_wf_link=True,
critical=True,
)
raise bp.error
if __name__ == "__main__":
logging.getLogger().setLevel(level=logging.INFO)
assert not is_shallow()
with stash():
if os.getenv("ROBOT_CLICKHOUSE_SSH_KEY", ""):
with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"):
main()
else:
main()