diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 76a94ed8dc0..6a3615fc5db 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -14,6 +14,7 @@ from s3_helper import S3Helper from pr_info import PRInfo from version_helper import ( ClickHouseVersion, + Git, get_version_from_repo, update_version_local, ) @@ -209,7 +210,7 @@ def main(): s3_helper = S3Helper("https://s3.amazonaws.com") - version = get_version_from_repo() + version = get_version_from_repo(git=Git(True)) release_or_pr = get_release_or_pr(pr_info, build_config, version) s3_path_prefix = "/".join((release_or_pr, pr_info.sha, build_name)) diff --git a/tests/ci/git_helper.py b/tests/ci/git_helper.py index 50414ffb470..0921ace35f6 100644 --- a/tests/ci/git_helper.py +++ b/tests/ci/git_helper.py @@ -13,6 +13,9 @@ TAG_REGEXP = ( ) SHA_REGEXP = r"\A([0-9]|[a-f]){40}\Z" +CWD = p.dirname(p.realpath(__file__)) +TWEAK = 1 + # Py 3.8 removeprefix and removesuffix def removeprefix(string: str, prefix: str): @@ -46,8 +49,8 @@ def release_branch(name: str): class Runner: """lightweight check_output wrapper with stripping last NEW_LINE""" - def __init__(self, cwd: str = p.dirname(p.realpath(__file__))): - self.cwd = cwd + def __init__(self, cwd: str = CWD): + self._cwd = cwd def run(self, cmd: str, cwd: Optional[str] = None) -> str: if cwd is None: @@ -56,22 +59,47 @@ class Runner: cmd, shell=True, cwd=cwd, encoding="utf-8" ).strip() + @property + def cwd(self) -> str: + return self._cwd + + @cwd.setter + def cwd(self, value: str): + # Set _cwd only once, then set it to readonly + if self._cwd != CWD: + return + self._cwd = value + + +git_runner = Runner() +# Set cwd to abs path of git root +git_runner.cwd = p.relpath( + p.join(git_runner.cwd, git_runner.run("git rev-parse --show-cdup")) +) + + +def get_tags() -> List[str]: + if git_runner.run("git rev-parse --is-shallow-repository") == "true": + raise RuntimeError("attempt to run on a shallow repository") + return git_runner.run("git tag").split() + class Git: """A small wrapper around subprocess to invoke git commands""" - def __init__(self): - runner = Runner() - rel_root = runner.run("git rev-parse --show-cdup") - self.root = p.realpath(p.join(runner.cwd, rel_root)) - self._tag_pattern = re.compile(TAG_REGEXP) - runner.cwd = self.root - self.run = runner.run + _tag_pattern = re.compile(TAG_REGEXP) + + def __init__(self, ignore_no_tags: bool = False): + self.root = git_runner.cwd + self._ignore_no_tags = ignore_no_tags + self.run = git_runner.run + self.latest_tag = "" + self.new_tag = "" self.new_branch = "" self.branch = "" self.sha = "" self.sha_short = "" - self.description = "" + self.description = "shallow-checkout" self.commits_since_tag = 0 self.update() @@ -82,6 +110,19 @@ class Git: self.sha_short = self.sha[:11] # The following command shows the most recent tag in a graph # Format should match TAG_REGEXP + if ( + self._ignore_no_tags + and self.run("git rev-parse --is-shallow-repository") == "true" + ): + try: + self._update_tags() + except subprocess.CalledProcessError: + pass + + return + self._update_tags() + + def _update_tags(self): self.latest_tag = self.run("git describe --tags --abbrev=0") # Format should be: {latest_tag}-{commits_since_tag}-g{sha_short} self.description = self.run("git describe --tags --long") @@ -89,10 +130,11 @@ class Git: self.run(f"git rev-list {self.latest_tag}..HEAD --count") ) - def check_tag(self, value: str): + @staticmethod + def check_tag(value: str): if value == "": return - if not self._tag_pattern.match(value): + if not Git._tag_pattern.match(value): raise ValueError(f"last tag {value} doesn't match the pattern") @property @@ -118,10 +160,7 @@ class Git: if not self.latest_tag.endswith("-testing"): # When we are on the tag, we still need to have tweak=1 to not # break cmake with versions like 12.13.14.0 - return self.commits_since_tag or 1 + return self.commits_since_tag or TWEAK version = self.latest_tag.split("-", maxsplit=1)[0] return int(version.split(".")[-1]) + self.commits_since_tag - - def get_tags(self) -> List[str]: - return self.run("git tag").split() diff --git a/tests/ci/git_test.py b/tests/ci/git_test.py index 785c9b62cce..3aedd8a8dea 100644 --- a/tests/ci/git_test.py +++ b/tests/ci/git_test.py @@ -4,7 +4,7 @@ from unittest.mock import patch import os.path as p import unittest -from git_helper import Git, Runner +from git_helper import Git, Runner, CWD class TestRunner(unittest.TestCase): @@ -19,6 +19,18 @@ class TestRunner(unittest.TestCase): output = runner.run("echo 1") self.assertEqual(output, "1") + def test_one_time_writeable_cwd(self): + runner = Runner() + self.assertEqual(runner.cwd, CWD) + runner.cwd = "/bin" + self.assertEqual(runner.cwd, "/bin") + runner.cwd = "/" + self.assertEqual(runner.cwd, "/bin") + runner = Runner("/") + self.assertEqual(runner.cwd, "/") + runner.cwd = "/bin" + self.assertEqual(runner.cwd, "/") + class TestGit(unittest.TestCase): def setUp(self): @@ -31,6 +43,7 @@ class TestGit(unittest.TestCase): self.addCleanup(update_patcher.stop) self.git = Git() update_mock.assert_called_once() + self.git.run("test") self.run_mock.assert_called_once() self.git.new_branch = "NEW_BRANCH_NAME" self.git.new_tag = "v21.12.333.22222-stable" diff --git a/tests/ci/release.py b/tests/ci/release.py index ca1a66dbbac..15a25fbd534 100755 --- a/tests/ci/release.py +++ b/tests/ci/release.py @@ -10,6 +10,7 @@ from git_helper import commit, release_branch from version_helper import ( FILE_WITH_VERSION_PATH, ClickHouseVersion, + Git, VersionType, get_abs_path, get_version_from_repo, @@ -17,6 +18,9 @@ from version_helper import ( ) +git = Git() + + class Repo: VALID = ("ssh", "https", "origin") @@ -53,8 +57,8 @@ class Release: self._release_commit = "" self.release_commit = release_commit self.release_type = release_type - self._version = get_version_from_repo() - self._git = self._version._git + self._git = git + self._version = get_version_from_repo(git=self._git) self._release_branch = "" self._rollback_stack = [] # type: List[str] @@ -75,7 +79,7 @@ class Release: def read_version(self): self._git.update() - self.version = get_version_from_repo() + self.version = get_version_from_repo(git=self._git) def check_prerequisites(self): """ diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index b5c4aa7434b..08ee052cf1a 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -2,9 +2,9 @@ import logging import os.path as p from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, ArgumentTypeError -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union -from git_helper import Git, removeprefix +from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt" CHANGELOG_IN_PATH = "debian/changelog.in" @@ -34,8 +34,6 @@ SET(VERSION_STRING {string}) # end of autochange """ -git = Git() - class ClickHouseVersion: """Immutable version class. On update returns a new instance""" @@ -46,7 +44,7 @@ class ClickHouseVersion: minor: Union[int, str], patch: Union[int, str], revision: Union[int, str], - git: Git, + git: Optional[Git], tweak: str = None, ): self._major = int(major) @@ -54,9 +52,11 @@ class ClickHouseVersion: self._patch = int(patch) self._revision = int(revision) self._git = git - self._tweak = None + self._tweak = TWEAK if tweak is not None: self._tweak = int(tweak) + elif self._git is not None: + self._tweak = self._git.tweak self._describe = "" def update(self, part: str) -> "ClickHouseVersion": @@ -91,7 +91,7 @@ class ClickHouseVersion: @property def tweak(self) -> int: - return self._tweak or self._git.tweak + return self._tweak @property def revision(self) -> int: @@ -99,7 +99,9 @@ class ClickHouseVersion: @property def githash(self) -> str: - return self._git.sha + if self._git is not None: + return self._git.sha + return "0000000000000000000000000000000000000000" @property def describe(self): @@ -171,7 +173,7 @@ def validate_version(version: str): def get_abs_path(path: str) -> str: - return p.abspath(p.join(git.root, path)) + return p.abspath(p.join(git_runner.cwd, path)) def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS: @@ -197,6 +199,7 @@ def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS: def get_version_from_repo( versions_path: str = FILE_WITH_VERSION_PATH, + git: Optional[Git] = None, ) -> ClickHouseVersion: versions = read_versions(versions_path) return ClickHouseVersion( @@ -208,14 +211,16 @@ def get_version_from_repo( ) -def get_version_from_string(version: str) -> ClickHouseVersion: +def get_version_from_string( + version: str, git: Optional[Git] = None +) -> ClickHouseVersion: validate_version(version) parts = version.split(".") return ClickHouseVersion(parts[0], parts[1], parts[2], -1, git, parts[3]) def get_version_from_tag(tag: str) -> ClickHouseVersion: - git.check_tag(tag) + Git.check_tag(tag) tag = tag[1:].split("-")[0] return get_version_from_string(tag) @@ -236,7 +241,7 @@ def version_arg(version: str) -> ClickHouseVersion: def get_tagged_versions() -> List[ClickHouseVersion]: versions = [] - for tag in git.get_tags(): + for tag in get_tags(): try: version = get_version_from_tag(tag) versions.append(version) @@ -261,19 +266,20 @@ def update_contributors( ): # Check if we have shallow checkout by comparing number of lines # '--is-shallow-repository' is in git since 2.15, 2017-10-30 - if git.run("git rev-parse --is-shallow-repository") == "true" and not force: + if git_runner.run("git rev-parse --is-shallow-repository") == "true" and not force: logging.warning("The repository is shallow, refusing to update contributors") if raise_error: raise RuntimeError("update_contributors executed on a shallow repository") return - contributors = git.run("git shortlog HEAD --summary") + # format: " 1016 Alexey Arno" + shortlog = git_runner.run("git shortlog HEAD --summary") contributors = sorted( - [c.split(maxsplit=1)[-1].replace('"', r"\"") for c in contributors.split("\n")], + [c.split(maxsplit=1)[-1].replace('"', r"\"") for c in shortlog.split("\n")], ) contributors = [f' "{c}",' for c in contributors] - executer = p.relpath(p.realpath(__file__), git.root) + executer = p.relpath(p.realpath(__file__), git_runner.cwd) content = CONTRIBUTORS_TEMPLATE.format( executer=executer, contributors="\n".join(contributors) )