thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 15:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #34740 from ClickHouse/release-stable

Browse Source 
Add a patch release for stable/lts packages to release.py
This commit is contained in:
Mikhail f. Shiryaev 2022-02-22 11:39:23 +01:00 committed by GitHub
commit c44aeda23c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 268 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tests/ci/git_helper.py
View File

@ -5,8 +5,13 @@ import re
import subprocess
from typing import Optional
TAG_REGEXP = r"^v\d{2}[.][1-9]\d*[.][1-9]\d*[.][1-9]\d*-(testing|prestable|stable|lts)$"
SHA_REGEXP = r"^([0-9]|[a-f]){40}$"
# ^ and $ match subline in `multiple\nlines`
# \A and \Z match only start and end of the whole string
RELEASE_BRANCH_REGEXP = r"\A\d+[.]\d+\Z"
TAG_REGEXP = (
r"\Av\d{2}[.][1-9]\d*[.][1-9]\d*[.][1-9]\d*-(testing|prestable|stable|lts)\Z"
)
SHA_REGEXP = r"\A([0-9]|[a-f]){40}\Z"
# Py 3.8 removeprefix and removesuffix
@ -31,6 +36,13 @@ def commit(name: str):
return name
def release_branch(name: str):
r = re.compile(RELEASE_BRANCH_REGEXP)
if not r.match(name):
raise argparse.ArgumentTypeError("release branch should be as 12.1")
return name
class Runner:
"""lightweight check_output wrapper with stripping last NEW_LINE"""

68
tests/ci/git_test.py Normal file
View File

@ -0,0 +1,68 @@
#!/usr/bin/env python
from unittest.mock import patch
import os.path as p
import unittest
from git_helper import Git, Runner
class TestRunner(unittest.TestCase):
def test_init(self):
runner = Runner()
self.assertEqual(runner.cwd, p.realpath(p.dirname(__file__)))
runner = Runner("/")
self.assertEqual(runner.cwd, "/")
def test_run(self):
runner = Runner()
output = runner.run("echo 1")
self.assertEqual(output, "1")
class TestGit(unittest.TestCase):
def setUp(self):
"""we use dummy git object"""
run_patcher = patch("git_helper.Runner.run", return_value="")
self.run_mock = run_patcher.start()
self.addCleanup(run_patcher.stop)
update_patcher = patch("git_helper.Git.update")
update_mock = update_patcher.start()
self.addCleanup(update_patcher.stop)
self.git = Git()
update_mock.assert_called_once()
self.run_mock.assert_called_once()
self.git.new_branch = "NEW_BRANCH_NAME"
self.git.new_tag = "v21.12.333.22222-stable"
self.git.branch = "old_branch"
self.git.sha = ""
self.git.sha_short = ""
self.git.latest_tag = ""
self.git.description = ""
self.git.commits_since_tag = 0
def test_tags(self):
self.git.new_tag = "v21.12.333.22222-stable"
self.git.latest_tag = "v21.12.333.22222-stable"
for tag_attr in ("new_tag", "latest_tag"):
self.assertEqual(getattr(self.git, tag_attr), "v21.12.333.22222-stable")
setattr(self.git, tag_attr, "")
self.assertEqual(getattr(self.git, tag_attr), "")
for tag in (
"v21.12.333-stable",
"v21.12.333-prestable",
"21.12.333.22222-stable",
"v21.12.333.22222-production",
):
with self.assertRaises(Exception):
setattr(self.git, tag_attr, tag)
def test_tweak(self):
self.git.commits_since_tag = 0
self.assertEqual(self.git.tweak, 1)
self.git.commits_since_tag = 2
self.assertEqual(self.git.tweak, 2)
self.git.latest_tag = "v21.12.333.22222-testing"
self.assertEqual(self.git.tweak, 22224)
self.git.commits_since_tag = 0
self.assertEqual(self.git.tweak, 22222)

262
tests/ci/release.py
View File

@ -6,7 +6,7 @@ from typing import List, Optional
import argparse
import logging
from git_helper import commit
from git_helper import commit, release_branch
from version_helper import (
FILE_WITH_VERSION_PATH,
ClickHouseVersion,
@ -18,14 +18,45 @@ from version_helper import (
)
class Repo:
VALID = ("ssh", "https", "origin")
def __init__(self, repo: str, protocol: str):
self._repo = repo
self._url = ""
self.url = protocol
@property
def url(self) -> str:
return self._url
@url.setter
def url(self, protocol: str):
if protocol == "ssh":
self._url = f"git@github.com:{self}.git"
elif protocol == "https":
self._url = f"https://github.com/{self}.git"
elif protocol == "origin":
self._url = protocol
else:
raise Exception(f"protocol must be in {self.VALID}")
def __str__(self):
return self._repo
class Release:
BIG = ("major", "minor")
SMALL = ("patch",)
def __init__(self, version: ClickHouseVersion):
self._version = version
self._git = version._git
def __init__(self, repo: Repo, release_commit: str, release_type: str):
self.repo = repo
self._release_commit = ""
self.release_commit = release_commit
self.release_type = release_type
self._version = get_version_from_repo()
self._git = self._version._git
self._release_branch = ""
self._rollback_stack = [] # type: List[str]
def run(self, cmd: str, cwd: Optional[str] = None) -> str:
@ -35,32 +66,45 @@ class Release:
logging.info("Running command%s:\n %s", cwd_text, cmd)
return self._git.run(cmd, cwd)
def update(self):
def set_release_branch(self):
# Get the actual version for the commit before check
with self._checkout(self.release_commit, True):
self.read_version()
self.release_branch = f"{self.version.major}.{self.version.minor}"
self.read_version()
def read_version(self):
self._git.update()
self.version = get_version_from_repo()
def do(self, args: argparse.Namespace):
self.release_commit = args.commit
def do(self, no_check_dirty: bool, no_check_branch: bool, no_prestable: bool):
if not args.no_check_dirty:
if not no_check_dirty:
logging.info("Checking if repo is clean")
self.run("git diff HEAD --exit-code")
if not args.no_check_branch:
self.check_branch(args.release_type)
self.set_release_branch()
if args.release_type in self.BIG:
# Checkout to the commit, it will provide the correct current version
with self._checkout(self.release_commit, True):
if args.no_prestable:
if not no_check_branch:
self.check_branch()
with self._checkout(self.release_commit, True):
if self.release_type in self.BIG:
# Checkout to the commit, it will provide the correct current version
if no_prestable:
logging.info("Skipping prestable stage")
else:
with self.prestable(args):
with self.prestable():
logging.info("Prestable part of the releasing is done")
with self.testing(args):
with self.testing():
logging.info("Testing part of the releasing is done")
elif self.release_type in self.SMALL:
with self.stable():
logging.info("Stable part of the releasing is done")
self.log_rollback()
def check_no_tags_after(self):
@ -71,19 +115,27 @@ class Release:
f"{tags_after_commit}\nChoose another commit"
)
def check_branch(self, release_type: str):
if release_type in self.BIG:
def check_branch(self):
if self.release_type in self.BIG:
# Commit to spin up the release must belong to a main branch
output = self.run(f"git branch --contains={self.release_commit} master")
if "master" not in output:
branch = "master"
output = self.run(f"git branch --contains={self.release_commit} {branch}")
if branch not in output:
raise Exception(
f"commit {self.release_commit} must belong to 'master' for "
f"{release_type} release"
f"commit {self.release_commit} must belong to {branch} for "
f"{self.release_type} release"
)
if release_type in self.SMALL:
branch = f"{self.version.major}.{self.version.minor}"
if self._git.branch != branch:
raise Exception(f"branch must be '{branch}' for {release_type} release")
return
elif self.release_type in self.SMALL:
output = self.run(
f"git branch --contains={self.release_commit} {self.release_branch}"
)
if self.release_branch not in output:
raise Exception(
f"commit {self.release_commit} must be in "
f"'{self.release_branch}' branch for {self.release_type} release"
)
return
def log_rollback(self):
if self._rollback_stack:
@ -95,31 +147,60 @@ class Release:
)
@contextmanager
def prestable(self, args: argparse.Namespace):
def prestable(self):
self.check_no_tags_after()
# Create release branch
self.update()
release_branch = f"{self.version.major}.{self.version.minor}"
with self._create_branch(release_branch, self.release_commit):
with self._checkout(release_branch, True):
self.update()
self.read_version()
with self._create_branch(self.release_branch, self.release_commit):
with self._checkout(self.release_branch, True):
self.read_version()
self.version.with_description(VersionType.PRESTABLE)
with self._create_gh_release(args):
with self._bump_prestable_version(release_branch, args):
with self._create_gh_release(True):
with self._bump_prestable_version():
# At this point everything will rollback automatically
yield
@contextmanager
def testing(self, args: argparse.Namespace):
def stable(self):
self.check_no_tags_after()
self.read_version()
version_type = VersionType.STABLE
if self.version.minor % 5 == 3: # our 3 and 8 are LTS
version_type = VersionType.LTS
self.version.with_description(version_type)
with self._create_gh_release(False):
self.version = self.version.update(self.release_type)
self.version.with_description(version_type)
update_cmake_version(self.version)
cmake_path = get_abs_path(FILE_WITH_VERSION_PATH)
# Checkouting the commit of the branch and not the branch itself,
# then we are able to skip rollback
with self._checkout(f"{self.release_branch}@{{0}}", False):
current_commit = self.run("git rev-parse HEAD")
self.run(
f"git commit -m "
f"'Update version to {self.version.string}' '{cmake_path}'"
)
with self._push(
"HEAD", with_rollback_on_fail=False, remote_ref=self.release_branch
):
# DO NOT PUT ANYTHING ELSE HERE
# The push must be the last action and mean the successful release
self._rollback_stack.append(
f"git push {self.repo.url} "
f"+{current_commit}:{self.release_branch}"
)
yield
@contextmanager
def testing(self):
# Create branch for a version bump
self.update()
self.version = self.version.update(args.release_type)
self.read_version()
self.version = self.version.update(self.release_type)
helper_branch = f"{self.version.major}.{self.version.minor}-prepare"
with self._create_branch(helper_branch, self.release_commit):
with self._checkout(helper_branch, True):
self.update()
self.version = self.version.update(args.release_type)
with self._bump_testing_version(helper_branch, args):
with self._bump_testing_version(helper_branch):
yield
@property
@ -132,6 +213,14 @@ class Release:
raise ValueError(f"version must be ClickHouseVersion, not {type(version)}")
self._version = version
@property
def release_branch(self) -> str:
return self._release_branch
@release_branch.setter
def release_branch(self, branch: str):
self._release_branch = release_branch(branch)
@property
def release_commit(self) -> str:
return self._release_commit
@ -141,7 +230,8 @@ class Release:
self._release_commit = commit(release_commit)
@contextmanager
def _bump_prestable_version(self, release_branch: str, args: argparse.Namespace):
def _bump_prestable_version(self):
# Update only git, origal version stays the same
self._git.update()
new_version = self.version.patch_update()
new_version.with_description("prestable")
@ -150,35 +240,38 @@ class Release:
self.run(
f"git commit -m 'Update version to {new_version.string}' '{cmake_path}'"
)
with self._push(release_branch, args):
with self._push(self.release_branch):
with self._create_gh_label(
f"v{release_branch}-must-backport", "10dbed", args
f"v{self.release_branch}-must-backport", "10dbed"
):
with self._create_gh_label(
f"v{release_branch}-affected", "c2bfff", args
f"v{self.release_branch}-affected", "c2bfff"
):
self.run(
f"gh pr create --repo {args.repo} --title 'Release pull "
f"request for branch {release_branch}' --head {release_branch} "
f"gh pr create --repo {self.repo} --title "
f"'Release pull request for branch {self.release_branch}' "
f"--head {self.release_branch} --label release "
"--body 'This PullRequest is a part of ClickHouse release "
"cycle. It is used by CI system only. Do not perform any "
"changes with it.' --label release"
"changes with it.'"
)
# Here the prestable part is done
yield
@contextmanager
def _bump_testing_version(self, helper_branch: str, args: argparse.Namespace):
def _bump_testing_version(self, helper_branch: str):
self.read_version()
self.version = self.version.update(self.release_type)
self.version.with_description("testing")
update_cmake_version(self.version)
cmake_path = get_abs_path(FILE_WITH_VERSION_PATH)
self.run(
f"git commit -m 'Update version to {self.version.string}' '{cmake_path}'"
)
with self._push(helper_branch, args):
with self._push(helper_branch):
body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md")
self.run(
f"gh pr create --repo {args.repo} --title 'Update version after "
f"gh pr create --repo {self.repo} --title 'Update version after "
f"release' --head {helper_branch} --body-file '{body_file}'"
)
# Here the prestable part is done
@ -216,9 +309,12 @@ class Release:
raise
@contextmanager
def _create_gh_label(self, label: str, color: str, args: argparse.Namespace):
self.run(f"gh api repos/{args.repo}/labels -f name={label} -f color={color}")
rollback_cmd = f"gh api repos/{args.repo}/labels/{label} -X DELETE"
def _create_gh_label(self, label: str, color_hex: str):
# API call, https://docs.github.com/en/rest/reference/issues#create-a-label
self.run(
f"gh api repos/{self.repo}/labels -f name={label} -f color={color_hex}"
)
rollback_cmd = f"gh api repos/{self.repo}/labels/{label} -X DELETE"
self._rollback_stack.append(rollback_cmd)
try:
yield
@ -228,15 +324,18 @@ class Release:
raise
@contextmanager
def _create_gh_release(self, args: argparse.Namespace):
with self._create_tag(args):
def _create_gh_release(self, as_prerelease: bool):
with self._create_tag():
# Preserve tag if version is changed
tag = self.version.describe
prerelease = ""
if as_prerelease:
prerelease = "--prerelease"
self.run(
f"gh release create --prerelease --draft --repo {args.repo} "
f"gh release create {prerelease} --draft --repo {self.repo} "
f"--title 'Release {tag}' '{tag}'"
)
rollback_cmd = f"gh release delete --yes --repo {args.repo} '{tag}'"
rollback_cmd = f"gh release delete --yes --repo {self.repo} '{tag}'"
self._rollback_stack.append(rollback_cmd)
try:
yield
@ -246,13 +345,13 @@ class Release:
raise
@contextmanager
def _create_tag(self, args: argparse.Namespace):
def _create_tag(self):
tag = self.version.describe
self.run(f"git tag -a -m 'Release {tag}' '{tag}'")
rollback_cmd = f"git tag -d '{tag}'"
self._rollback_stack.append(rollback_cmd)
try:
with self._push(f"'{tag}'", args):
with self._push(f"'{tag}'"):
yield
except BaseException:
logging.warning("Rolling back tag %s", tag)
@ -260,15 +359,22 @@ class Release:
raise
@contextmanager
def _push(self, ref: str, args: argparse.Namespace):
self.run(f"git push git@github.com:{args.repo}.git {ref}")
rollback_cmd = f"git push -d git@github.com:{args.repo}.git {ref}"
self._rollback_stack.append(rollback_cmd)
def _push(self, ref: str, with_rollback_on_fail: bool = True, remote_ref: str = ""):
if remote_ref == "":
remote_ref = ref
self.run(f"git push {self.repo.url} {ref}:{remote_ref}")
if with_rollback_on_fail:
rollback_cmd = f"git push -d {self.repo.url} {remote_ref}"
self._rollback_stack.append(rollback_cmd)
try:
yield
except BaseException:
logging.warning("Rolling back pushed ref %s", ref)
self.run(rollback_cmd)
if with_rollback_on_fail:
logging.warning("Rolling back pushed ref %s", ref)
self.run(rollback_cmd)
raise
@ -284,10 +390,17 @@ def parse_args() -> argparse.Namespace:
default="ClickHouse/ClickHouse",
help="repository to create the release",
)
parser.add_argument(
"--remote-protocol",
"-p",
default="ssh",
choices=Repo.VALID,
help="repo protocol for git commands remote, 'origin' is a special case and "
"uses 'origin' as a remote",
)
parser.add_argument(
"--type",
default="minor",
# choices=Release.BIG+Release.SMALL, # add support later
choices=Release.BIG + Release.SMALL,
dest="release_type",
help="a release type, new branch is created only for 'major' and 'minor'",
@ -307,29 +420,26 @@ def parse_args() -> argparse.Namespace:
parser.add_argument(
"--no-check-dirty",
action="store_true",
help="skip check repository for uncommited changes",
help="(dangerous) skip check repository for uncommited changes",
)
parser.add_argument(
"--no-check-branch",
action="store_true",
help="by default, 'major' and 'minor' types work only for master, and 'patch' "
"works only for a release branches, that name should be the same as "
"'$MAJOR.$MINOR' version, e.g. 22.2",
help="(debug or development only) by default, 'major' and 'minor' types work "
"only for master, and 'patch' works only for a release branches, that name "
"should be the same as '$MAJOR.$MINOR' version, e.g. 22.2",
)
return parser.parse_args()
def prestable():
pass
def main():
logging.basicConfig(level=logging.INFO)
args = parse_args()
release = Release(get_version_from_repo())
repo = Repo(args.repo, args.remote_protocol)
release = Release(repo, args.commit, args.release_type)
release.do(args)
release.do(args.no_check_dirty, args.no_check_branch, args.no_prestable)
if __name__ == "__main__":