ClickHouse/tests/ci/release.py

501 lines
18 KiB
Python
Raw Normal View History

2022-02-14 21:48:01 +00:00
#!/usr/bin/env python
from contextlib import contextmanager
from typing import List, Optional
2022-02-14 21:48:01 +00:00
import argparse
import logging
import subprocess
2022-02-14 21:48:01 +00:00
from git_helper import commit, release_branch
2022-02-14 21:48:01 +00:00
from version_helper import (
FILE_WITH_VERSION_PATH,
2022-04-21 15:29:20 +00:00
GENERATED_CONTRIBUTORS,
2022-02-14 21:48:01 +00:00
ClickHouseVersion,
Git,
2022-02-14 21:48:01 +00:00
VersionType,
get_abs_path,
get_version_from_repo,
update_cmake_version,
2022-04-21 15:29:20 +00:00
update_contributors,
2022-02-14 21:48:01 +00:00
)
git = Git()
class Repo:
VALID = ("ssh", "https", "origin")
def __init__(self, repo: str, protocol: str):
self._repo = repo
self._url = ""
self.url = protocol
@property
def url(self) -> str:
return self._url
@url.setter
def url(self, protocol: str):
if protocol == "ssh":
self._url = f"git@github.com:{self}.git"
elif protocol == "https":
self._url = f"https://github.com/{self}.git"
elif protocol == "origin":
self._url = protocol
else:
raise Exception(f"protocol must be in {self.VALID}")
def __str__(self):
return self._repo
2022-02-14 21:48:01 +00:00
class Release:
BIG = ("major", "minor")
SMALL = ("patch",)
2022-04-21 15:29:20 +00:00
CMAKE_PATH = get_abs_path(FILE_WITH_VERSION_PATH)
CONTRIBUTORS_PATH = get_abs_path(GENERATED_CONTRIBUTORS)
2022-02-14 21:48:01 +00:00
def __init__(self, repo: Repo, release_commit: str, release_type: str):
self.repo = repo
2022-02-14 21:48:01 +00:00
self._release_commit = ""
self.release_commit = release_commit
self.release_type = release_type
self._git = git
self._version = get_version_from_repo(git=self._git)
self._release_branch = ""
self._rollback_stack = [] # type: List[str]
2022-02-14 21:48:01 +00:00
def run(self, cmd: str, cwd: Optional[str] = None) -> str:
2022-02-16 16:04:25 +00:00
cwd_text = ""
if cwd:
cwd_text = f" (CWD='{cwd}')"
logging.info("Running command%s:\n %s", cwd_text, cmd)
2022-02-14 21:48:01 +00:00
return self._git.run(cmd, cwd)
def set_release_branch(self):
# Fetch release commit in case it does not exist locally
self.run(f"git fetch {self.repo.url} {self.release_commit}")
# Get the actual version for the commit before check
with self._checkout(self.release_commit, True):
self.read_version()
self.release_branch = f"{self.version.major}.{self.version.minor}"
self.read_version()
def read_version(self):
2022-02-16 14:59:51 +00:00
self._git.update()
self.version = get_version_from_repo(git=self._git)
2022-02-14 21:48:01 +00:00
def get_stable_release_type(self) -> str:
if self.version.minor % 5 == 3: # our 3 and 8 are LTS
return VersionType.LTS
return VersionType.STABLE
2022-02-25 10:53:21 +00:00
def check_prerequisites(self):
"""
Check tooling installed in the system
"""
self.run("gh auth status")
self.run("git status")
def do(self, check_dirty: bool, check_branch: bool, with_release_branch: bool):
2022-02-25 10:53:21 +00:00
self.check_prerequisites()
2022-02-14 21:48:01 +00:00
2022-02-21 11:44:37 +00:00
if check_dirty:
2022-02-16 14:59:51 +00:00
logging.info("Checking if repo is clean")
try:
self.run("git diff HEAD --exit-code")
except subprocess.CalledProcessError:
logging.fatal("Repo contains uncommitted changes")
raise
2022-02-14 21:48:01 +00:00
self.set_release_branch()
2022-02-21 11:44:37 +00:00
if check_branch:
self.check_branch()
2022-02-16 14:59:51 +00:00
with self._checkout(self.release_commit, True):
if self.release_type in self.BIG:
# Checkout to the commit, it will provide the correct current version
if with_release_branch:
with self.create_release_branch():
logging.info("Prestable part of the releasing is done")
2022-03-17 15:50:20 +00:00
else:
logging.info("Skipping creating release branch stage")
2022-02-16 14:59:51 +00:00
rollback = self._rollback_stack.copy()
try:
with self.testing():
logging.info("Testing part of the releasing is done")
except (Exception, KeyboardInterrupt):
logging.fatal("Testing part failed, rollback previous steps")
rollback.reverse()
for cmd in rollback:
self.run(cmd)
raise
2022-02-14 21:48:01 +00:00
2022-02-18 22:36:40 +00:00
elif self.release_type in self.SMALL:
with self.stable():
logging.info("Stable part of the releasing is done")
self.log_rollback()
2022-02-14 21:48:01 +00:00
def check_no_tags_after(self):
tags_after_commit = self.run(f"git tag --contains={self.release_commit}")
if tags_after_commit:
raise Exception(
f"Commit {self.release_commit} belongs to following tags:\n"
f"{tags_after_commit}\nChoose another commit"
)
def check_branch(self):
branch = self.release_branch
if self.release_type in self.BIG:
2022-02-14 21:48:01 +00:00
# Commit to spin up the release must belong to a main branch
2022-02-18 11:35:24 +00:00
branch = "master"
elif self.release_type not in self.SMALL:
raise (
ValueError(
f"release_type {self.release_type} neiter in {self.BIG} nor "
f"in {self.SMALL}"
2022-02-14 21:48:01 +00:00
)
)
# Prefetch the branch to have it updated
2022-05-19 08:05:34 +00:00
if self._git.branch == branch:
self.run("git pull")
else:
self.run(f"git fetch {self.repo.url} {branch}:{branch}")
output = self.run(f"git branch --contains={self.release_commit} {branch}")
if branch not in output:
raise Exception(
f"commit {self.release_commit} must belong to {branch} "
f"for {self.release_type} release"
)
2022-02-14 21:48:01 +00:00
def log_rollback(self):
if self._rollback_stack:
rollback = self._rollback_stack
rollback.reverse()
logging.info(
"To rollback the action run the following commands:\n %s",
"\n ".join(rollback),
)
2022-02-14 21:48:01 +00:00
@contextmanager
def create_release_branch(self):
2022-02-14 21:48:01 +00:00
self.check_no_tags_after()
# Create release branch
self.read_version()
with self._create_branch(self.release_branch, self.release_commit):
with self._checkout(self.release_branch, True):
self.read_version()
self.version.with_description(self.get_stable_release_type())
with self._create_gh_release(False):
with self._bump_release_branch():
2022-02-14 21:48:01 +00:00
# At this point everything will rollback automatically
yield
2022-02-18 22:36:40 +00:00
@contextmanager
def stable(self):
self.check_no_tags_after()
self.read_version()
version_type = self.get_stable_release_type()
2022-02-18 22:36:40 +00:00
self.version.with_description(version_type)
with self._create_gh_release(False):
self.version = self.version.update(self.release_type)
self.version.with_description(version_type)
update_cmake_version(self.version)
2022-04-21 15:29:20 +00:00
update_contributors(raise_error=True)
2022-02-18 22:36:40 +00:00
# Checkouting the commit of the branch and not the branch itself,
# then we are able to skip rollback
with self._checkout(f"{self.release_branch}^0", False):
2022-02-18 22:36:40 +00:00
current_commit = self.run("git rev-parse HEAD")
self.run(
f"git commit -m "
2022-04-21 15:29:20 +00:00
f"'Update version to {self.version.string}' "
f"'{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'"
2022-02-18 22:36:40 +00:00
)
with self._push(
"HEAD", with_rollback_on_fail=False, remote_ref=self.release_branch
):
# DO NOT PUT ANYTHING ELSE HERE
# The push must be the last action and mean the successful release
self._rollback_stack.append(
f"git push {self.repo.url} "
f"+{current_commit}:{self.release_branch}"
)
yield
2022-02-14 21:48:01 +00:00
@contextmanager
def testing(self):
2022-02-14 21:48:01 +00:00
# Create branch for a version bump
self.read_version()
self.version = self.version.update(self.release_type)
2022-02-14 21:48:01 +00:00
helper_branch = f"{self.version.major}.{self.version.minor}-prepare"
2022-02-16 14:59:51 +00:00
with self._create_branch(helper_branch, self.release_commit):
2022-02-14 21:48:01 +00:00
with self._checkout(helper_branch, True):
with self._bump_testing_version(helper_branch):
2022-02-14 21:48:01 +00:00
yield
2022-02-16 14:59:51 +00:00
@property
def version(self) -> ClickHouseVersion:
return self._version
@version.setter
def version(self, version: ClickHouseVersion):
if not isinstance(version, ClickHouseVersion):
raise ValueError(f"version must be ClickHouseVersion, not {type(version)}")
self._version = version
@property
def release_branch(self) -> str:
return self._release_branch
@release_branch.setter
def release_branch(self, branch: str):
self._release_branch = release_branch(branch)
2022-02-16 14:59:51 +00:00
@property
def release_commit(self) -> str:
return self._release_commit
@release_commit.setter
def release_commit(self, release_commit: str):
self._release_commit = commit(release_commit)
2022-02-14 21:48:01 +00:00
@contextmanager
def _bump_release_branch(self):
# Update only git, origal version stays the same
self._git.update()
2022-02-14 21:48:01 +00:00
new_version = self.version.patch_update()
version_type = self.get_stable_release_type()
pr_labels = "--label release"
if version_type == VersionType.LTS:
pr_labels += " --label release-lts"
new_version.with_description(version_type)
2022-02-14 21:48:01 +00:00
update_cmake_version(new_version)
2022-04-21 15:29:20 +00:00
update_contributors(raise_error=True)
2022-02-14 21:48:01 +00:00
self.run(
2022-04-21 15:29:20 +00:00
f"git commit -m 'Update version to {new_version.string}' "
f"'{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'"
2022-02-14 21:48:01 +00:00
)
with self._push(self.release_branch):
with self._create_gh_label(
f"v{self.release_branch}-must-backport", "10dbed"
):
with self._create_gh_label(
f"v{self.release_branch}-affected", "c2bfff"
):
# The following command is rolled back by self._push
self.run(
f"gh pr create --repo {self.repo} --title "
f"'Release pull request for branch {self.release_branch}' "
f"--head {self.release_branch} {pr_labels} "
"--body 'This PullRequest is a part of ClickHouse release "
"cycle. It is used by CI system only. Do not perform any "
"changes with it.'"
)
# Here the release branch part is done
yield
2022-02-14 21:48:01 +00:00
2022-02-16 14:59:51 +00:00
@contextmanager
def _bump_testing_version(self, helper_branch: str):
self.read_version()
2022-02-18 22:36:40 +00:00
self.version = self.version.update(self.release_type)
self.version.with_description(VersionType.TESTING)
2022-02-16 14:59:51 +00:00
update_cmake_version(self.version)
2022-04-21 15:29:20 +00:00
update_contributors(raise_error=True)
2022-02-16 14:59:51 +00:00
self.run(
2022-04-21 15:29:20 +00:00
f"git commit -m 'Update version to {self.version.string}' "
f"'{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'"
2022-02-16 14:59:51 +00:00
)
with self._push(helper_branch):
2022-02-16 14:59:51 +00:00
body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md")
self.run(
f"gh pr create --repo {self.repo} --title 'Update version after "
2022-02-16 14:59:51 +00:00
f"release' --head {helper_branch} --body-file '{body_file}'"
)
# Here the testing part is done
2022-02-16 14:59:51 +00:00
yield
@contextmanager
def _checkout(self, ref: str, with_checkout_back: bool = False):
2022-02-16 14:59:51 +00:00
orig_ref = self._git.branch or self._git.sha
need_rollback = False
if ref not in (self._git.branch, self._git.sha):
need_rollback = True
self.run(f"git checkout {ref}")
# checkout is not put into rollback_stack intentionally
rollback_cmd = f"git checkout {orig_ref}"
2022-02-16 14:59:51 +00:00
try:
yield
except (Exception, KeyboardInterrupt):
2022-02-16 14:59:51 +00:00
logging.warning("Rolling back checked out %s for %s", ref, orig_ref)
self.run(f"git reset --hard; git checkout {orig_ref}")
raise
else:
if with_checkout_back and need_rollback:
self.run(rollback_cmd)
2022-02-16 14:59:51 +00:00
@contextmanager
def _create_branch(self, name: str, start_point: str = ""):
self.run(f"git branch {name} {start_point}")
rollback_cmd = f"git branch -D {name}"
self._rollback_stack.append(rollback_cmd)
2022-02-16 14:59:51 +00:00
try:
yield
except (Exception, KeyboardInterrupt):
2022-02-16 14:59:51 +00:00
logging.warning("Rolling back created branch %s", name)
self.run(rollback_cmd)
2022-02-16 14:59:51 +00:00
raise
@contextmanager
def _create_gh_label(self, label: str, color_hex: str):
# API call, https://docs.github.com/en/rest/reference/issues#create-a-label
self.run(
f"gh api repos/{self.repo}/labels -f name={label} -f color={color_hex}"
)
rollback_cmd = f"gh api repos/{self.repo}/labels/{label} -X DELETE"
self._rollback_stack.append(rollback_cmd)
2022-02-16 14:59:51 +00:00
try:
yield
except (Exception, KeyboardInterrupt):
2022-02-16 16:04:25 +00:00
logging.warning("Rolling back label %s", label)
self.run(rollback_cmd)
2022-02-16 16:04:25 +00:00
raise
2022-02-16 14:59:51 +00:00
2022-02-14 21:48:01 +00:00
@contextmanager
2022-02-18 22:36:40 +00:00
def _create_gh_release(self, as_prerelease: bool):
with self._create_tag():
2022-02-14 21:48:01 +00:00
# Preserve tag if version is changed
tag = self.version.describe
2022-02-18 22:36:40 +00:00
prerelease = ""
if as_prerelease:
prerelease = "--prerelease"
2022-02-14 21:48:01 +00:00
self.run(
2022-05-18 11:10:12 +00:00
f"gh release create {prerelease} --repo {self.repo} "
f"--title 'Release {tag}' '{tag}'"
2022-02-14 21:48:01 +00:00
)
rollback_cmd = f"gh release delete --yes --repo {self.repo} '{tag}'"
self._rollback_stack.append(rollback_cmd)
2022-02-14 21:48:01 +00:00
try:
yield
except (Exception, KeyboardInterrupt):
2022-02-14 21:48:01 +00:00
logging.warning("Rolling back release publishing")
self.run(rollback_cmd)
2022-02-14 21:48:01 +00:00
raise
@contextmanager
def _create_tag(self):
2022-02-14 21:48:01 +00:00
tag = self.version.describe
self.run(f"git tag -a -m 'Release {tag}' '{tag}'")
rollback_cmd = f"git tag -d '{tag}'"
self._rollback_stack.append(rollback_cmd)
2022-02-14 21:48:01 +00:00
try:
with self._push(f"'{tag}'"):
2022-02-14 21:48:01 +00:00
yield
except (Exception, KeyboardInterrupt):
2022-02-14 21:48:01 +00:00
logging.warning("Rolling back tag %s", tag)
self.run(rollback_cmd)
2022-02-14 21:48:01 +00:00
raise
@contextmanager
2022-02-18 22:36:40 +00:00
def _push(self, ref: str, with_rollback_on_fail: bool = True, remote_ref: str = ""):
if remote_ref == "":
remote_ref = ref
self.run(f"git push {self.repo.url} {ref}:{remote_ref}")
if with_rollback_on_fail:
2022-02-18 22:36:40 +00:00
rollback_cmd = f"git push -d {self.repo.url} {remote_ref}"
self._rollback_stack.append(rollback_cmd)
2022-02-14 21:48:01 +00:00
try:
yield
except (Exception, KeyboardInterrupt):
if with_rollback_on_fail:
logging.warning("Rolling back pushed ref %s", ref)
self.run(rollback_cmd)
2022-02-14 21:48:01 +00:00
raise
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Script to release a new ClickHouse version, requires `git` and "
"`gh` (github-cli) commands",
)
parser.add_argument(
"--commit",
required=True,
type=commit,
help="commit create a release",
)
2022-02-14 21:48:01 +00:00
parser.add_argument(
"--repo",
default="ClickHouse/ClickHouse",
help="repository to create the release",
)
parser.add_argument(
"--remote-protocol",
"-p",
default="ssh",
choices=Repo.VALID,
help="repo protocol for git commands remote, 'origin' is a special case and "
"uses 'origin' as a remote",
)
2022-02-14 21:48:01 +00:00
parser.add_argument(
"--type",
default="minor",
choices=Release.BIG + Release.SMALL,
dest="release_type",
help="a release type, new branch is created only for 'major' and 'minor'",
)
parser.add_argument("--with-release-branch", default=True, help=argparse.SUPPRESS)
2022-02-21 11:44:37 +00:00
parser.add_argument(
"--no-release-branch",
dest="with_release_branch",
2022-02-21 11:44:37 +00:00
action="store_false",
default=argparse.SUPPRESS,
help=f"if set, for release types in {Release.BIG} skip creating release branch",
2022-02-21 11:44:37 +00:00
)
parser.add_argument("--check-dirty", default=True, help=argparse.SUPPRESS)
2022-02-14 21:48:01 +00:00
parser.add_argument(
"--no-check-dirty",
2022-02-21 11:44:37 +00:00
dest="check_dirty",
action="store_false",
default=argparse.SUPPRESS,
help="(dangerous) if set, skip check repository for uncommited changes",
2022-02-14 21:48:01 +00:00
)
2022-02-21 11:44:37 +00:00
parser.add_argument("--check-branch", default=True, help=argparse.SUPPRESS)
2022-02-14 21:48:01 +00:00
parser.add_argument(
"--no-check-branch",
2022-02-21 11:44:37 +00:00
dest="check_branch",
action="store_false",
default=argparse.SUPPRESS,
help="(debug or development only, dangerous) if set, skip the branch check for "
"a run. By default, 'major' and 'minor' types workonly for master, and 'patch' "
"works only for a release branches, that name "
"should be the same as '$MAJOR.$MINOR' version, e.g. 22.2",
2022-02-14 21:48:01 +00:00
)
return parser.parse_args()
def main():
logging.basicConfig(level=logging.INFO)
args = parse_args()
repo = Repo(args.repo, args.remote_protocol)
release = Release(repo, args.commit, args.release_type)
2022-02-14 21:48:01 +00:00
release.do(args.check_dirty, args.check_branch, args.with_release_branch)
2022-02-14 21:48:01 +00:00
if __name__ == "__main__":
main()