thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Backport #64759 to 24.3: Adjust the version_helper and script to a new release scheme

Browse Source
This commit is contained in:
robot-clickhouse 2024-06-11 16:07:57 +00:00
parent e6b9c7620a
commit c876299f33
8 changed files with 256 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tests/ci/ci_config.py
View File

@ -262,6 +262,7 @@ builds_job_config = JobConfig(
"./packages",
"./docker/packager/packager",
"./rust",
"./tests/ci/version_helper.py",
# FIXME: This is a WA to rebuild the CH and recreate the Performance.tar.zst artifact
# when there are changes in performance test scripts.
# Due to the current design of the perf test we need to rebuild CH when the performance test changes,

86
tests/ci/git_helper.py
View File

@ -4,7 +4,8 @@ import logging
import os.path as p
import re
import subprocess
from typing import Any, List, Optional
import tempfile
from typing import Any, List, Literal, Optional
logger = logging.getLogger(__name__)
@ -12,7 +13,9 @@ logger = logging.getLogger(__name__)
# \A and \Z match only start and end of the whole string
RELEASE_BRANCH_REGEXP = r"\A\d+[.]\d+\Z"
TAG_REGEXP = (
r"\Av\d{2}[.][1-9]\d*[.][1-9]\d*[.][1-9]\d*-(testing|prestable|stable|lts)\Z"
r"\Av\d{2}" # First two digits of major part
r"([.][1-9]\d*){3}" # minor.patch.tweak parts
r"-(new|testing|prestable|stable|lts)\Z" # suffix with a version type
)
SHA_REGEXP = re.compile(r"\A([0-9]|[a-f]){40}\Z")
@ -115,17 +118,35 @@ class Git:
_tag_pattern = re.compile(TAG_REGEXP)
def __init__(self, ignore_no_tags: bool = False):
"""
new_tag is used for special v24.1.1.1-new tags where the previous version is moved to the release branch
* 66666666666 Some another commit with version 24.8.1.xxxxx-testing, tweak is counted from new_tag = v24.8.1.1-new
| * 55555555555 (tag: v24.7.1.123123123-stable, branch: 24.7) tweak counted from new_tag = v24.7.1.1-new
|/
* 44444444444 (tag: v24.8.1.1-new)
| * 33333333333 (tag: v24.6.1.123123123-stable, branch: 24.6) tweak counted from new_tag = v24.6.1.1-new
|/
* 22222222222 (tag: v24.7.1.1-new)
| * 11111111111 (tag: v24.5.1.123123123-stable, branch: 24.5) tweak counted from new_tag = v24.4.1.2088-stable
|/
* 00000000000 (tag: v24.6.1.1-new)
* 6d4b31322d1 (tag: v24.4.1.2088-stable)
* 2c5c589a882 (tag: v24.3.1.2672-lts)
* 891689a4150 (tag: v24.2.1.2248-stable)
* 5a024dfc093 (tag: v24.1.1.2048-stable)
* a2faa65b080 (tag: v23.12.1.1368-stable)
* 05bc8ef1e02 (tag: v23.11.1.2711-stable)
"""
self.root = git_runner.cwd
self._ignore_no_tags = ignore_no_tags
self.run = git_runner.run
self.latest_tag = ""
self.new_tag = ""
self.new_branch = ""
self.branch = ""
self.sha = ""
self.sha_short = ""
self.description = "shallow-checkout"
self.commits_since_tag = 0
self.commits_since_latest = 0
self.commits_since_new = 0
self.update()
def update(self):
@ -148,10 +169,20 @@ class Git:
stderr = subprocess.DEVNULL if suppress_stderr else None
self.latest_tag = self.run("git describe --tags --abbrev=0", stderr=stderr)
# Format should be: {latest_tag}-{commits_since_tag}-g{sha_short}
self.description = self.run("git describe --tags --long")
self.commits_since_tag = int(
self.commits_since_latest = int(
self.run(f"git rev-list {self.latest_tag}..HEAD --count")
)
if self.latest_tag.endswith("-new"):
# We won't change the behaviour of the the "latest_tag"
# So here we set "new_tag" to the previous tag in the graph, that will allow
# getting alternative "tweak"
self.new_tag = self.run(
f"git describe --tags --abbrev=0 --exclude='{self.latest_tag}'",
stderr=stderr,
)
self.commits_since_new = int(
self.run(f"git rev-list {self.new_tag}..HEAD --count")
)
@staticmethod
def check_tag(value: str) -> None:
@ -180,19 +211,34 @@ class Git:
@property
def tweak(self) -> int:
if not self.latest_tag.endswith("-testing"):
return self._tweak("latest")
@property
def tweak_to_new(self) -> int:
return self._tweak("new")
def _tweak(self, tag_type: Literal["latest", "new"]) -> int:
"""Accepts latest or new as a tag_type and returns the tweak number to it"""
if tag_type == "latest":
commits = self.commits_since_latest
tag = self.latest_tag
else:
commits = self.commits_since_new
tag = self.new_tag
if not tag.endswith("-testing"):
# When we are on the tag, we still need to have tweak=1 to not
# break cmake with versions like 12.13.14.0
if not self.commits_since_tag:
# We are in a tagged commit. The tweak should match the
# current version's value
version = self.latest_tag.split("-", maxsplit=1)[0]
try:
return int(version.split(".")[-1])
except ValueError:
# There are no tags, or a wrong tag. Return default
return TWEAK
return self.commits_since_tag
if commits:
return commits
# We are in a tagged commit or shallow checkout. The tweak should match the
# current version's value
version = tag.split("-", maxsplit=1)[0]
try:
return int(version.split(".")[-1])
except ValueError:
# There are no tags (shallow checkout), or a wrong tag. Return default
return TWEAK
version = self.latest_tag.split("-", maxsplit=1)[0]
return int(version.split(".")[-1]) + self.commits_since_tag
version = tag.split("-", maxsplit=1)[0]
return int(version.split(".")[-1]) + commits

47
tests/ci/release.py
View File

@ -86,6 +86,7 @@ class Release:
self._version = get_version_from_repo(git=self._git)
self.release_version = self.version
self._release_branch = ""
self._version_new_tag = None # type: Optional[ClickHouseVersion]
self._rollback_stack = [] # type: List[str]
def run(
@ -172,7 +173,8 @@ class Release:
)
raise
self.check_commit_release_ready()
if self.release_type == self.PATCH:
self.check_commit_release_ready()
def do(
self, check_dirty: bool, check_run_from_master: bool, check_branch: bool
@ -320,10 +322,16 @@ class Release:
self.check_no_tags_after()
# Create release branch
self.read_version()
with self._create_branch(self.release_branch, self.release_commit):
with self._checkout(self.release_branch, True):
with self._bump_release_branch():
yield
assert self._version_new_tag is not None
with self._create_tag(
self._version_new_tag.describe,
self.release_commit,
f"Initial commit for release {self._version_new_tag.major}.{self._version_new_tag.minor}",
):
with self._create_branch(self.release_branch, self.release_commit):
with self._checkout(self.release_branch, True):
with self._bump_release_branch():
yield
@contextmanager
def patch_release(self):
@ -436,6 +444,11 @@ class Release:
self.version.with_description(VersionType.TESTING)
self._update_cmake_contributors(self.version)
self._commit_cmake_contributors(self.version)
# Create a version-new tag
self._version_new_tag = self.version.copy()
self._version_new_tag.tweak = 1
self._version_new_tag.with_description(VersionType.NEW)
with self._push(helper_branch):
body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md")
# The following command is rolled back by deleting branch in self._push
@ -450,10 +463,10 @@ class Release:
@contextmanager
def _checkout(self, ref: str, with_checkout_back: bool = False) -> Iterator[None]:
self._git.update()
orig_ref = self._git.branch or self._git.sha
need_rollback = False
rollback_cmd = ""
if ref not in (self._git.branch, self._git.sha):
need_rollback = True
self.run(f"git checkout {ref}")
# checkout is not put into rollback_stack intentionally
rollback_cmd = f"git checkout {orig_ref}"
@ -466,7 +479,7 @@ class Release:
self.run(f"git reset --hard; git checkout -f {orig_ref}")
raise
# Normal flow when we need to checkout back
if with_checkout_back and need_rollback:
if with_checkout_back and rollback_cmd:
self.run(rollback_cmd)
@contextmanager
@ -502,9 +515,9 @@ class Release:
@contextmanager
def _create_gh_release(self, as_prerelease: bool) -> Iterator[None]:
with self._create_tag():
tag = self.release_version.describe
with self._create_tag(tag, self.release_commit):
# Preserve tag if version is changed
tag = self.release_version.describe
prerelease = ""
if as_prerelease:
prerelease = "--prerelease"
@ -526,13 +539,13 @@ class Release:
raise
@contextmanager
def _create_tag(self):
tag = self.release_version.describe
self.run(
f"git tag -a -m 'Release {tag}' '{tag}' {self.release_commit}",
dry_run=self.dry_run,
)
rollback_cmd = f"{self.dry_run_prefix}git tag -d '{tag}'"
def _create_tag(
self, tag: str, commit: str, tag_message: str = ""
) -> Iterator[None]:
tag_message = tag_message or "Release {tag}"
# Create tag even in dry-run
self.run(f"git tag -a -m '{tag_message}' '{tag}' {commit}")
rollback_cmd = f"git tag -d '{tag}'"
self._rollback_stack.append(rollback_cmd)
try:
with self._push(tag):

52
tests/ci/test_git.py
View File

@ -1,10 +1,11 @@
#!/usr/bin/env python
from unittest.mock import patch
import os.path as p
import unittest
from dataclasses import dataclass
from unittest.mock import patch
from git_helper import Git, Runner, CWD
from git_helper import CWD, Git, Runner, git_runner
class TestRunner(unittest.TestCase):
@ -35,8 +36,10 @@ class TestRunner(unittest.TestCase):
class TestGit(unittest.TestCase):
def setUp(self):
"""we use dummy git object"""
# get the git_runner's cwd to set it properly before the Runner is patched
_ = git_runner.cwd
run_patcher = patch("git_helper.Runner.run", return_value="")
self.run_mock = run_patcher.start()
run_mock = run_patcher.start()
self.addCleanup(run_patcher.stop)
update_patcher = patch("git_helper.Git.update")
update_mock = update_patcher.start()
@ -44,15 +47,13 @@ class TestGit(unittest.TestCase):
self.git = Git()
update_mock.assert_called_once()
self.git.run("test")
self.run_mock.assert_called_once()
self.git.new_branch = "NEW_BRANCH_NAME"
self.git.new_tag = "v21.12.333.22222-stable"
run_mock.assert_called_once()
self.git.branch = "old_branch"
self.git.sha = ""
self.git.sha_short = ""
self.git.latest_tag = ""
self.git.description = ""
self.git.commits_since_tag = 0
self.git.commits_since_latest = 0
self.git.commits_since_new = 0
def test_tags(self):
self.git.new_tag = "v21.12.333.22222-stable"
@ -71,11 +72,30 @@ class TestGit(unittest.TestCase):
setattr(self.git, tag_attr, tag)
def test_tweak(self):
self.git.commits_since_tag = 0
self.assertEqual(self.git.tweak, 1)
self.git.commits_since_tag = 2
self.assertEqual(self.git.tweak, 2)
self.git.latest_tag = "v21.12.333.22222-testing"
self.assertEqual(self.git.tweak, 22224)
self.git.commits_since_tag = 0
self.assertEqual(self.git.tweak, 22222)
# tweak for the latest tag
@dataclass
class TestCase:
tag: str
commits: int
tweak: int
cases = (
TestCase("", 0, 1),
TestCase("", 2, 2),
TestCase("v21.12.333.22222-stable", 0, 22222),
TestCase("v21.12.333.22222-stable", 2, 2),
TestCase("v21.12.333.22222-testing", 0, 22222),
TestCase("v21.12.333.22222-testing", 2, 22224),
)
for tag, commits, tweak in (
("latest_tag", "commits_since_latest", "tweak"),
("new_tag", "commits_since_new", "tweak_to_new"),
):
for tc in cases:
setattr(self.git, tag, tc.tag)
setattr(self.git, commits, tc.commits)
self.assertEqual(
getattr(self.git, tweak),
tc.tweak,
f"Wrong tweak for tag {tc.tag} and commits {tc.commits} of {tag}",
)

57
tests/ci/test_version.py
View File

@ -2,8 +2,13 @@
import unittest
from argparse import ArgumentTypeError
from dataclasses import dataclass
from pathlib import Path
import version_helper as vh
from git_helper import Git
CHV = vh.ClickHouseVersion
class TestFunctions(unittest.TestCase):
@ -32,3 +37,55 @@ class TestFunctions(unittest.TestCase):
for error_case in error_cases:
with self.assertRaises(ArgumentTypeError):
version = vh.version_arg(error_case[0])
def test_get_version_from_repo(self):
@dataclass
class TestCase:
latest_tag: str
commits_since_latest: int
new_tag: str
commits_since_new: int
expected: CHV
cases = (
TestCase(
"v24.6.1.1-new",
15,
"v24.4.1.2088-stable",
415,
CHV(24, 5, 1, 54487, None, 415),
),
TestCase(
"v24.6.1.1-testing",
15,
"v24.4.1.2088-stable",
415,
CHV(24, 5, 1, 54487, None, 16),
),
TestCase(
"v24.6.1.1-stable",
15,
"v24.4.1.2088-stable",
415,
CHV(24, 5, 1, 54487, None, 15),
),
TestCase(
"v24.5.1.1-stable",
15,
"v24.4.1.2088-stable",
415,
CHV(24, 5, 1, 54487, None, 15),
),
)
git = Git(True)
for tc in cases:
git.latest_tag = tc.latest_tag
git.commits_since_latest = tc.commits_since_latest
git.new_tag = tc.new_tag
git.commits_since_new = tc.commits_since_new
self.assertEqual(
vh.get_version_from_repo(
Path("tests/ci/tests/autogenerated_versions.txt"), git
),
tc.expected,
)

12
tests/ci/tests/autogenerated_versions.txt Normal file
View File

@ -0,0 +1,12 @@
# This variables autochanged by tests/ci/version_helper.py:
# NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
# only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
SET(VERSION_REVISION 54487)
SET(VERSION_MAJOR 24)
SET(VERSION_MINOR 5)
SET(VERSION_PATCH 1)
SET(VERSION_GITHASH 70a1d3a63d47f0be077d67b8deb907230fc7cfb0)
SET(VERSION_DESCRIBE v24.5.1.1-testing)
SET(VERSION_STRING 24.5.1.1)
# end of autochange

1
tests/ci/tmp/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
*

88
tests/ci/version_helper.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3
import logging
import os.path as p
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, ArgumentTypeError
from pathlib import Path
from typing import Any, Dict, Iterable, List, Literal, Optional, Set, Tuple, Union
from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix
@ -22,7 +22,7 @@ VERSIONS = Dict[str, Union[int, str]]
VERSIONS_TEMPLATE = """# This variables autochanged by tests/ci/version_helper.py:
# NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
# NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION,
# only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
SET(VERSION_REVISION {revision})
SET(VERSION_MAJOR {major})
@ -47,7 +47,7 @@ class ClickHouseVersion:
patch: Union[int, str],
revision: Union[int, str],
git: Optional[Git],
tweak: Optional[str] = None,
tweak: Optional[Union[int, str]] = None,
):
self._major = int(major)
self._minor = int(minor)
@ -95,7 +95,7 @@ class ClickHouseVersion:
if self._git is not None:
self._git.update()
return ClickHouseVersion(
self.major, self.minor, self.patch, self.revision, self._git, "1"
self.major, self.minor, self.patch, self.revision, self._git, 1
)
@property
@ -114,6 +114,10 @@ class ClickHouseVersion:
def tweak(self) -> int:
return self._tweak
@tweak.setter
def tweak(self, tweak: int) -> None:
self._tweak = tweak
@property
def revision(self) -> int:
return self._revision
@ -172,7 +176,7 @@ class ClickHouseVersion:
self.patch,
self.revision,
self._git,
str(self.tweak),
self.tweak,
)
try:
copy.with_description(self.description)
@ -190,7 +194,9 @@ class ClickHouseVersion:
and self.tweak == other.tweak
)
def __lt__(self, other: "ClickHouseVersion") -> bool:
def __lt__(self, other: Any) -> bool:
if not isinstance(self, type(other)):
return NotImplemented
for part in ("major", "minor", "patch", "tweak"):
if getattr(self, part) < getattr(other, part):
return True
@ -220,10 +226,11 @@ ClickHouseVersions = List[ClickHouseVersion]
class VersionType:
LTS = "lts"
NEW = "new"
PRESTABLE = "prestable"
STABLE = "stable"
TESTING = "testing"
VALID = (TESTING, PRESTABLE, STABLE, LTS)
VALID = (NEW, TESTING, PRESTABLE, STABLE, LTS)
def validate_version(version: str) -> None:
@ -234,43 +241,56 @@ def validate_version(version: str) -> None:
int(part)
def get_abs_path(path: str) -> str:
return p.abspath(p.join(git_runner.cwd, path))
def get_abs_path(path: Union[Path, str]) -> Path:
return (Path(git_runner.cwd) / path).absolute()
def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS:
def read_versions(versions_path: Union[Path, str] = FILE_WITH_VERSION_PATH) -> VERSIONS:
versions = {}
path_to_file = get_abs_path(versions_path)
with open(path_to_file, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line.startswith("SET("):
continue
for line in get_abs_path(versions_path).read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line.startswith("SET("):
continue
value = 0 # type: Union[int, str]
name, value = line[4:-1].split(maxsplit=1)
name = removeprefix(name, "VERSION_").lower()
try:
value = int(value)
except ValueError:
pass
versions[name] = value
value = 0 # type: Union[int, str]
name, value = line[4:-1].split(maxsplit=1)
name = removeprefix(name, "VERSION_").lower()
try:
value = int(value)
except ValueError:
pass
versions[name] = value
return versions
def get_version_from_repo(
versions_path: str = FILE_WITH_VERSION_PATH,
versions_path: Union[Path, str] = FILE_WITH_VERSION_PATH,
git: Optional[Git] = None,
) -> ClickHouseVersion:
"""Get a ClickHouseVersion from FILE_WITH_VERSION_PATH. When the `git` parameter is
present, a proper `tweak` version part is calculated for case if the latest tag has
a `new` type and greater than version in `FILE_WITH_VERSION_PATH`"""
versions = read_versions(versions_path)
return ClickHouseVersion(
cmake_version = ClickHouseVersion(
versions["major"],
versions["minor"],
versions["patch"],
versions["revision"],
git,
)
# Since 24.5 we have tags like v24.6.1.1-new, and we must check if the release
# branch already has it's own commit. It's necessary for a proper tweak version
if git is not None and git.latest_tag:
version_from_tag = get_version_from_tag(git.latest_tag)
if (
version_from_tag.description == VersionType.NEW
and cmake_version < version_from_tag
):
# We are in a new release branch without existing release.
# We should change the tweak version to a `tweak_to_new`
cmake_version.tweak = git.tweak_to_new
return cmake_version
def get_version_from_string(
@ -350,15 +370,15 @@ def get_supported_versions(
def update_cmake_version(
version: ClickHouseVersion,
versions_path: str = FILE_WITH_VERSION_PATH,
versions_path: Union[Path, str] = FILE_WITH_VERSION_PATH,
) -> None:
path_to_file = get_abs_path(versions_path)
with open(path_to_file, "w", encoding="utf-8") as f:
f.write(VERSIONS_TEMPLATE.format_map(version.as_dict()))
get_abs_path(versions_path).write_text(
VERSIONS_TEMPLATE.format_map(version.as_dict()), encoding="utf-8"
)
def update_contributors(
relative_contributors_path: str = GENERATED_CONTRIBUTORS,
relative_contributors_path: Union[Path, str] = GENERATED_CONTRIBUTORS,
force: bool = False,
raise_error: bool = False,
) -> None:
@ -377,13 +397,11 @@ def update_contributors(
)
contributors = [f' "{c}",' for c in contributors]
executer = p.relpath(p.realpath(__file__), git_runner.cwd)
executer = Path(__file__).relative_to(git_runner.cwd)
content = CONTRIBUTORS_TEMPLATE.format(
executer=executer, contributors="\n".join(contributors)
)
contributors_path = get_abs_path(relative_contributors_path)
with open(contributors_path, "w", encoding="utf-8") as cfd:
cfd.write(content)
get_abs_path(relative_contributors_path).write_text(content, encoding="utf-8")
def update_version_local(version, version_type="testing"):