ClickHouse/tests/ci/version_helper.py

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

503 lines
15 KiB
Python
Raw Normal View History

2021-10-21 13:18:59 +00:00
#!/usr/bin/env python3
2022-01-28 13:39:23 +00:00
import logging
2024-01-08 12:24:27 +00:00
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, ArgumentTypeError
2024-06-04 17:55:11 +00:00
from pathlib import Path
from typing import Any, Dict, Iterable, List, Literal, Optional, Set, Tuple, Union
2022-01-28 13:39:23 +00:00
2024-01-08 12:24:27 +00:00
from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix
2021-10-21 13:18:59 +00:00
FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt"
CHANGELOG_IN_PATH = "debian/changelog.in"
CHANGELOG_PATH = "debian/changelog"
2022-01-28 13:39:23 +00:00
GENERATED_CONTRIBUTORS = "src/Storages/System/StorageSystemContributors.generated.cpp"
# It has {{ for plain "{"
CONTRIBUTORS_TEMPLATE = """// autogenerated by {executer}
const char * auto_contributors[] {{
{contributors}
nullptr}};
"""
VERSIONS = Dict[str, Union[int, str]]
2022-09-05 13:59:20 +00:00
VERSIONS_TEMPLATE = """# This variables autochanged by tests/ci/version_helper.py:
2022-01-28 13:39:23 +00:00
# NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION,
2022-01-28 13:39:23 +00:00
# only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
SET(VERSION_REVISION {revision})
SET(VERSION_MAJOR {major})
SET(VERSION_MINOR {minor})
SET(VERSION_PATCH {patch})
SET(VERSION_GITHASH {githash})
SET(VERSION_DESCRIBE {describe})
SET(VERSION_STRING {string})
# end of autochange
"""
2021-10-21 13:18:59 +00:00
2022-01-27 12:21:57 +00:00
class ClickHouseVersion:
2022-01-28 13:39:23 +00:00
"""Immutable version class. On update returns a new instance"""
2024-01-08 12:24:27 +00:00
PART_TYPE = Literal["major", "minor", "patch"]
2022-01-28 13:39:23 +00:00
def __init__(
self,
major: Union[int, str],
minor: Union[int, str],
patch: Union[int, str],
revision: Union[int, str],
git: Optional[Git],
tweak: Optional[Union[int, str]] = None,
2022-01-28 13:39:23 +00:00
):
self._major = int(major)
self._minor = int(minor)
self._patch = int(patch)
self._revision = int(revision)
self._git = git
self._tweak = TWEAK
if tweak is not None:
self._tweak = int(tweak)
elif self._git is not None:
self._tweak = self._git.tweak
2022-01-28 13:39:23 +00:00
self._describe = ""
2023-02-27 13:18:41 +00:00
self._description = ""
2021-10-21 13:18:59 +00:00
2024-01-08 12:24:27 +00:00
def update(self, part: PART_TYPE) -> "ClickHouseVersion":
2022-01-28 13:39:23 +00:00
"""If part is valid, returns a new version"""
2022-11-10 15:47:44 +00:00
if part == "major":
return self.major_update()
if part == "minor":
return self.minor_update()
if part == "patch":
return self.patch_update()
raise KeyError(f"wrong part {part} is used")
2021-10-21 13:18:59 +00:00
def bump(self) -> "ClickHouseVersion":
if self.minor < 12:
self._minor += 1
self._revision += 1
self._patch = 1
self._tweak = 1
else:
self._major += 1
self._revision += 1
self._patch = 1
self._tweak = 1
return self
2022-01-28 13:39:23 +00:00
def major_update(self) -> "ClickHouseVersion":
2022-08-03 09:43:47 +00:00
if self._git is not None:
self._git.update()
2022-01-28 13:39:23 +00:00
return ClickHouseVersion(self.major + 1, 1, 1, self.revision + 1, self._git)
def minor_update(self) -> "ClickHouseVersion":
2022-08-03 09:43:47 +00:00
if self._git is not None:
self._git.update()
2021-10-21 13:18:59 +00:00
return ClickHouseVersion(
2022-01-28 13:39:23 +00:00
self.major, self.minor + 1, 1, self.revision + 1, self._git
2022-01-27 12:21:57 +00:00
)
2021-10-21 13:18:59 +00:00
2022-01-28 13:39:23 +00:00
def patch_update(self) -> "ClickHouseVersion":
2022-08-03 09:43:47 +00:00
if self._git is not None:
self._git.update()
2021-10-21 13:18:59 +00:00
return ClickHouseVersion(
2022-01-28 13:39:23 +00:00
self.major, self.minor, self.patch + 1, self.revision, self._git
2022-01-27 12:21:57 +00:00
)
2021-10-21 13:18:59 +00:00
def reset_tweak(self) -> "ClickHouseVersion":
if self._git is not None:
self._git.update()
return ClickHouseVersion(
self.major, self.minor, self.patch, self.revision, self._git, 1
)
2022-01-28 13:39:23 +00:00
@property
def major(self) -> int:
return self._major
@property
def minor(self) -> int:
return self._minor
@property
def patch(self) -> int:
return self._patch
@property
def tweak(self) -> int:
return self._tweak
2022-01-28 13:39:23 +00:00
@tweak.setter
def tweak(self, tweak: int) -> None:
self._tweak = tweak
2022-01-28 13:39:23 +00:00
@property
def revision(self) -> int:
return self._revision
@property
def githash(self) -> str:
"returns the CURRENT git SHA1"
if self._git is not None:
return self._git.sha
return "0000000000000000000000000000000000000000"
2022-01-28 13:39:23 +00:00
@property
def describe(self):
return self._describe
2023-02-27 13:18:41 +00:00
@property
def description(self) -> str:
return self._description
2022-01-28 13:39:23 +00:00
@property
def string(self):
2022-01-27 12:21:57 +00:00
return ".".join(
2022-01-28 13:39:23 +00:00
(str(self.major), str(self.minor), str(self.patch), str(self.tweak))
2022-01-27 12:21:57 +00:00
)
2021-10-21 13:18:59 +00:00
2023-11-30 13:06:52 +00:00
@property
def is_lts(self) -> bool:
"""our X.3 and X.8 are LTS"""
return self.minor % 5 == 3
2024-06-27 17:43:10 +00:00
def get_stable_release_type(self) -> str:
if self.is_lts:
return VersionType.LTS
return VersionType.STABLE
2022-01-28 13:39:23 +00:00
def as_dict(self) -> VERSIONS:
return {
"revision": self.revision,
"major": self.major,
"minor": self.minor,
"patch": self.patch,
"tweak": self.tweak,
"githash": self.githash,
"describe": self.describe,
"string": self.string,
}
def as_tuple(self) -> Tuple[int, int, int, int]:
2021-10-21 13:18:59 +00:00
return (self.major, self.minor, self.patch, self.tweak)
2022-01-28 13:39:23 +00:00
def with_description(self, version_type):
if version_type not in VersionType.VALID:
raise ValueError(f"version type {version_type} not in {VersionType.VALID}")
2023-02-27 13:18:41 +00:00
self._description = version_type
2022-01-28 13:39:23 +00:00
self._describe = f"v{self.string}-{version_type}"
2024-06-27 17:43:10 +00:00
return self
2022-01-28 13:39:23 +00:00
def copy(self) -> "ClickHouseVersion":
copy = ClickHouseVersion(
self.major,
self.minor,
self.patch,
self.revision,
self._git,
self.tweak,
)
try:
copy.with_description(self.description)
except ValueError:
pass
return copy
2022-11-10 15:47:44 +00:00
def __eq__(self, other: Any) -> bool:
if not isinstance(self, type(other)):
return NotImplemented
return bool(
self.major == other.major
and self.minor == other.minor
and self.patch == other.patch
and self.tweak == other.tweak
)
def __lt__(self, other: Any) -> bool:
if not isinstance(self, type(other)):
return NotImplemented
for part in ("major", "minor", "patch", "tweak"):
if getattr(self, part) < getattr(other, part):
return True
elif getattr(self, part) > getattr(other, part):
return False
return False
def __le__(self, other: "ClickHouseVersion") -> bool:
return self == other or self < other
def __hash__(self):
return hash(self.__repr__)
def __str__(self):
return f"{self.string}"
def __repr__(self):
return (
f"<ClickHouseVersion({self.major},{self.minor},{self.patch},{self.tweak},"
f"'{self.description}')>"
)
ClickHouseVersions = List[ClickHouseVersion]
2021-10-21 13:18:59 +00:00
2022-01-27 12:21:57 +00:00
class VersionType:
2022-01-28 13:39:23 +00:00
LTS = "lts"
NEW = "new"
2022-01-28 13:39:23 +00:00
PRESTABLE = "prestable"
2021-10-21 13:18:59 +00:00
STABLE = "stable"
TESTING = "testing"
VALID = (NEW, TESTING, PRESTABLE, STABLE, LTS)
2022-01-28 13:39:23 +00:00
2022-11-10 15:47:44 +00:00
def validate_version(version: str) -> None:
parts = version.split(".")
if len(parts) != 4:
raise ValueError(f"{version} does not contain 4 parts")
for part in parts:
int(part)
2024-06-04 17:55:11 +00:00
def get_abs_path(path: Union[Path, str]) -> Path:
return (Path(git_runner.cwd) / path).absolute()
2022-01-28 13:39:23 +00:00
2024-06-04 17:55:11 +00:00
def read_versions(versions_path: Union[Path, str] = FILE_WITH_VERSION_PATH) -> VERSIONS:
2022-01-28 13:39:23 +00:00
versions = {}
2024-06-04 17:55:11 +00:00
for line in get_abs_path(versions_path).read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line.startswith("SET("):
continue
value = 0 # type: Union[int, str]
name, value = line[4:-1].split(maxsplit=1)
name = removeprefix(name, "VERSION_").lower()
try:
value = int(value)
except ValueError:
pass
versions[name] = value
2021-10-21 13:18:59 +00:00
2022-01-28 13:39:23 +00:00
return versions
2021-10-21 13:18:59 +00:00
2022-01-28 13:39:23 +00:00
def get_version_from_repo(
2024-06-04 17:55:11 +00:00
versions_path: Union[Path, str] = FILE_WITH_VERSION_PATH,
git: Optional[Git] = None,
2022-01-28 13:39:23 +00:00
) -> ClickHouseVersion:
"""Get a ClickHouseVersion from FILE_WITH_VERSION_PATH. When the `git` parameter is
present, a proper `tweak` version part is calculated for case if the latest tag has
a `new` type and greater than version in `FILE_WITH_VERSION_PATH`"""
2022-01-28 13:39:23 +00:00
versions = read_versions(versions_path)
cmake_version = ClickHouseVersion(
2022-01-28 13:39:23 +00:00
versions["major"],
versions["minor"],
versions["patch"],
versions["revision"],
git,
2021-10-21 13:18:59 +00:00
)
# Since 24.5 we have tags like v24.6.1.1-new, and we must check if the release
# branch already has it's own commit. It's necessary for a proper tweak version
if git is not None and git.latest_tag:
version_from_tag = get_version_from_tag(git.latest_tag)
if (
version_from_tag.description == VersionType.NEW
and cmake_version < version_from_tag
):
# We are in a new release branch without existing release.
# We should change the tweak version to a `tweak_to_new`
cmake_version.tweak = git.tweak_to_new
return cmake_version
2021-10-21 13:18:59 +00:00
def get_version_from_string(
version: str, git: Optional[Git] = None
) -> ClickHouseVersion:
validate_version(version)
parts = version.split(".")
return ClickHouseVersion(parts[0], parts[1], parts[2], -1, git, parts[3])
def get_version_from_tag(tag: str) -> ClickHouseVersion:
Git.check_tag(tag)
tag, description = tag[1:].split("-", 1)
version = get_version_from_string(tag)
version.with_description(description)
return version
def version_arg(version: str) -> ClickHouseVersion:
version = removeprefix(version, "refs/tags/")
try:
return get_version_from_string(version)
except ValueError:
pass
try:
return get_version_from_tag(version)
except ValueError:
pass
raise ArgumentTypeError(f"version {version} does not match tag of plain version")
def get_tagged_versions() -> ClickHouseVersions:
versions = []
for tag in get_tags():
try:
version = get_version_from_tag(tag)
versions.append(version)
except Exception:
continue
return sorted(versions)
def get_supported_versions(
versions: Optional[Iterable[ClickHouseVersion]] = None,
) -> Set[ClickHouseVersion]:
supported_stable = set() # type: Set[ClickHouseVersion]
supported_lts = set() # type: Set[ClickHouseVersion]
if versions:
versions = list(versions)
else:
# checks that repo is not shallow in background
versions = get_tagged_versions()
versions.sort()
versions.reverse()
for version in versions:
if len(supported_stable) < 3:
if not {
sv
for sv in supported_stable
if version.major == sv.major and version.minor == sv.minor
}:
supported_stable.add(version)
2023-11-30 13:06:52 +00:00
if (version.description == VersionType.LTS or version.is_lts) and len(
supported_lts
) < 2:
if not {
sv
for sv in supported_lts
if version.major == sv.major and version.minor == sv.minor
}:
supported_lts.add(version)
if len(supported_stable) == 3 and len(supported_lts) == 2:
break
return supported_lts.union(supported_stable)
2022-01-28 13:39:23 +00:00
def update_cmake_version(
version: ClickHouseVersion,
2024-06-04 17:55:11 +00:00
versions_path: Union[Path, str] = FILE_WITH_VERSION_PATH,
2022-11-10 15:47:44 +00:00
) -> None:
2024-06-04 17:55:11 +00:00
get_abs_path(versions_path).write_text(
VERSIONS_TEMPLATE.format_map(version.as_dict()), encoding="utf-8"
)
2022-01-28 13:39:23 +00:00
def update_contributors(
2024-06-04 17:55:11 +00:00
relative_contributors_path: Union[Path, str] = GENERATED_CONTRIBUTORS,
force: bool = False,
raise_error: bool = False,
2022-11-10 15:47:44 +00:00
) -> None:
2022-01-28 13:39:23 +00:00
# Check if we have shallow checkout by comparing number of lines
# '--is-shallow-repository' is in git since 2.15, 2017-10-30
if git_runner.run("git rev-parse --is-shallow-repository") == "true" and not force:
2022-01-28 13:39:23 +00:00
logging.warning("The repository is shallow, refusing to update contributors")
if raise_error:
raise RuntimeError("update_contributors executed on a shallow repository")
2022-01-28 13:39:23 +00:00
return
# format: " 1016 Alexey Arno"
shortlog = git_runner.run("git shortlog HEAD --summary")
2024-04-05 23:48:50 +00:00
escaping = str.maketrans({"\\": "\\\\", '"': '\\"'})
2022-01-28 13:39:23 +00:00
contributors = sorted(
2024-04-05 23:48:50 +00:00
[c.split(maxsplit=1)[-1].translate(escaping) for c in shortlog.split("\n")],
2022-01-27 12:21:57 +00:00
)
2022-01-28 13:39:23 +00:00
contributors = [f' "{c}",' for c in contributors]
2024-06-04 17:55:11 +00:00
executer = Path(__file__).relative_to(git_runner.cwd)
2022-01-28 13:39:23 +00:00
content = CONTRIBUTORS_TEMPLATE.format(
executer=executer, contributors="\n".join(contributors)
)
2024-06-04 17:55:11 +00:00
get_abs_path(relative_contributors_path).write_text(content, encoding="utf-8")
2021-10-21 13:18:59 +00:00
2022-01-27 12:21:57 +00:00
def update_version_local(version, version_type="testing"):
2022-01-28 13:39:23 +00:00
update_contributors()
version.with_description(version_type)
2022-03-24 16:50:31 +00:00
update_cmake_version(version)
2022-01-28 13:39:23 +00:00
def main():
"""The simplest thing it does - reads versions from cmake and produce the
environment variables that may be sourced in bash scripts"""
parser = ArgumentParser(
formatter_class=ArgumentDefaultsHelpFormatter,
description="The script reads versions from cmake and produce ENV variables",
)
parser.add_argument(
"--version-path",
"-p",
default=FILE_WITH_VERSION_PATH,
help="relative path to the cmake file with versions",
)
parser.add_argument(
"--version-type",
"-t",
choices=VersionType.VALID,
default=VersionType.TESTING,
2022-01-28 13:39:23 +00:00
help="optional parameter to generate DESCRIBE",
)
parser.add_argument(
"--export",
"-e",
action="store_true",
help="if the ENV variables should be exported",
)
parser.add_argument(
"--update-part",
2022-01-28 13:39:23 +00:00
choices=("major", "minor", "patch"),
help="the version part to update, tweak is always calculated from commits, "
"implies `--update-cmake`",
)
parser.add_argument(
"--update-cmake",
"-u",
action="store_true",
help=f"is update for {FILE_WITH_VERSION_PATH} is needed or not",
2022-01-28 13:39:23 +00:00
)
parser.add_argument(
"--update-contributors",
"-c",
action="store_true",
help=f"update {GENERATED_CONTRIBUTORS} file and exit, "
"doesn't work on shallow repo",
)
args = parser.parse_args()
if args.update_contributors:
update_contributors()
return
version = get_version_from_repo(args.version_path, Git(True))
2022-01-28 13:39:23 +00:00
if args.update_part:
version = version.update(args.update_part)
2022-01-28 13:39:23 +00:00
version.with_description(args.version_type)
2022-01-28 13:39:23 +00:00
if args.update_part or args.update_cmake:
2022-01-28 13:39:23 +00:00
update_cmake_version(version)
for k, v in version.as_dict().items():
name = f"CLICKHOUSE_VERSION_{k.upper()}"
print(f"{name}='{v}'")
if args.export:
print(f"export {name}")
if __name__ == "__main__":
main()