Adjust git_helper and version_helper for shallow clone

tests/ci/build_check.py

@@ -14,6 +14,7 @@ from s3_helper import S3Helper
 from pr_info import PRInfo
 from version_helper import (
     ClickHouseVersion,
+    Git,
     get_version_from_repo,
     update_version_local,
 )
@@ -209,7 +210,7 @@ def main():
 
     s3_helper = S3Helper("https://s3.amazonaws.com")
 
-    version = get_version_from_repo()
+    version = get_version_from_repo(git=Git(True))
     release_or_pr = get_release_or_pr(pr_info, build_config, version)
 
     s3_path_prefix = "/".join((release_or_pr, pr_info.sha, build_name))

tests/ci/git_helper.py

@@ -13,6 +13,9 @@ TAG_REGEXP = (
 )
 SHA_REGEXP = r"\A([0-9]|[a-f]){40}\Z"
 
+CWD = p.dirname(p.realpath(__file__))
+TWEAK = 1
+
 
 # Py 3.8 removeprefix and removesuffix
 def removeprefix(string: str, prefix: str):
@@ -46,8 +49,8 @@ def release_branch(name: str):
 class Runner:
     """lightweight check_output wrapper with stripping last NEW_LINE"""
 
-    def __init__(self, cwd: str = p.dirname(p.realpath(__file__))):
-        self.cwd = cwd
+    def __init__(self, cwd: str = CWD):
+        self._cwd = cwd
 
     def run(self, cmd: str, cwd: Optional[str] = None) -> str:
         if cwd is None:
@@ -56,22 +59,47 @@ class Runner:
             cmd, shell=True, cwd=cwd, encoding="utf-8"
         ).strip()
 
+    @property
+    def cwd(self) -> str:
+        return self._cwd
+
+    @cwd.setter
+    def cwd(self, value: str):
+        # Set _cwd only once, then set it to readonly
+        if self._cwd != CWD:
+            return
+        self._cwd = value
+
+
+git_runner = Runner()
+# Set cwd to abs path of git root
+git_runner.cwd = p.relpath(
+    p.join(git_runner.cwd, git_runner.run("git rev-parse --show-cdup"))
+)
+
+
+def get_tags() -> List[str]:
+    if git_runner.run("git rev-parse --is-shallow-repository") == "true":
+        raise RuntimeError("attempt to run on a shallow repository")
+    return git_runner.run("git tag").split()
+
 
 class Git:
     """A small wrapper around subprocess to invoke git commands"""
 
-    def __init__(self):
-        runner = Runner()
-        rel_root = runner.run("git rev-parse --show-cdup")
-        self.root = p.realpath(p.join(runner.cwd, rel_root))
-        self._tag_pattern = re.compile(TAG_REGEXP)
-        runner.cwd = self.root
-        self.run = runner.run
+    _tag_pattern = re.compile(TAG_REGEXP)
+
+    def __init__(self, ignore_no_tags: bool = False):
+        self.root = git_runner.cwd
+        self._ignore_no_tags = ignore_no_tags
+        self.run = git_runner.run
+        self.latest_tag = ""
+        self.new_tag = ""
         self.new_branch = ""
         self.branch = ""
         self.sha = ""
         self.sha_short = ""
-        self.description = ""
+        self.description = "shallow-checkout"
         self.commits_since_tag = 0
         self.update()
 
@@ -82,6 +110,19 @@ class Git:
         self.sha_short = self.sha[:11]
         # The following command shows the most recent tag in a graph
         # Format should match TAG_REGEXP
+        if (
+            self._ignore_no_tags
+            and self.run("git rev-parse --is-shallow-repository") == "true"
+        ):
+            try:
+                self._update_tags()
+            except subprocess.CalledProcessError:
+                pass
+
+            return
+        self._update_tags()
+
+    def _update_tags(self):
         self.latest_tag = self.run("git describe --tags --abbrev=0")
         # Format should be: {latest_tag}-{commits_since_tag}-g{sha_short}
         self.description = self.run("git describe --tags --long")
@@ -89,10 +130,11 @@ class Git:
             self.run(f"git rev-list {self.latest_tag}..HEAD --count")
         )
 
-    def check_tag(self, value: str):
+    @staticmethod
+    def check_tag(value: str):
         if value == "":
             return
-        if not self._tag_pattern.match(value):
+        if not Git._tag_pattern.match(value):
             raise ValueError(f"last tag {value} doesn't match the pattern")
 
     @property
@@ -118,10 +160,7 @@ class Git:
         if not self.latest_tag.endswith("-testing"):
             # When we are on the tag, we still need to have tweak=1 to not
             # break cmake with versions like 12.13.14.0
-            return self.commits_since_tag or 1
+            return self.commits_since_tag or TWEAK
 
         version = self.latest_tag.split("-", maxsplit=1)[0]
         return int(version.split(".")[-1]) + self.commits_since_tag
-
-    def get_tags(self) -> List[str]:
-        return self.run("git tag").split()

tests/ci/git_test.py

@@ -4,7 +4,7 @@ from unittest.mock import patch
 import os.path as p
 import unittest
 
-from git_helper import Git, Runner
+from git_helper import Git, Runner, CWD
 
 
 class TestRunner(unittest.TestCase):
@@ -19,6 +19,18 @@ class TestRunner(unittest.TestCase):
         output = runner.run("echo 1")
         self.assertEqual(output, "1")
 
+    def test_one_time_writeable_cwd(self):
+        runner = Runner()
+        self.assertEqual(runner.cwd, CWD)
+        runner.cwd = "/bin"
+        self.assertEqual(runner.cwd, "/bin")
+        runner.cwd = "/"
+        self.assertEqual(runner.cwd, "/bin")
+        runner = Runner("/")
+        self.assertEqual(runner.cwd, "/")
+        runner.cwd = "/bin"
+        self.assertEqual(runner.cwd, "/")
+
 
 class TestGit(unittest.TestCase):
     def setUp(self):
@@ -31,6 +43,7 @@ class TestGit(unittest.TestCase):
         self.addCleanup(update_patcher.stop)
         self.git = Git()
         update_mock.assert_called_once()
+        self.git.run("test")
         self.run_mock.assert_called_once()
         self.git.new_branch = "NEW_BRANCH_NAME"
         self.git.new_tag = "v21.12.333.22222-stable"

tests/ci/release.py

@@ -10,6 +10,7 @@ from git_helper import commit, release_branch
 from version_helper import (
     FILE_WITH_VERSION_PATH,
     ClickHouseVersion,
+    Git,
     VersionType,
     get_abs_path,
     get_version_from_repo,
@@ -17,6 +18,9 @@ from version_helper import (
 )
 
 
+git = Git()
+
+
 class Repo:
     VALID = ("ssh", "https", "origin")
 
@@ -53,8 +57,8 @@ class Release:
         self._release_commit = ""
         self.release_commit = release_commit
         self.release_type = release_type
-        self._version = get_version_from_repo()
-        self._git = self._version._git
+        self._git = git
+        self._version = get_version_from_repo(git=self._git)
         self._release_branch = ""
         self._rollback_stack = []  # type: List[str]
 
@@ -75,7 +79,7 @@ class Release:
 
     def read_version(self):
         self._git.update()
-        self.version = get_version_from_repo()
+        self.version = get_version_from_repo(git=self._git)
 
     def check_prerequisites(self):
         """

tests/ci/version_helper.py

@@ -2,9 +2,9 @@
 import logging
 import os.path as p
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, ArgumentTypeError
-from typing import Dict, List, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 
-from git_helper import Git, removeprefix
+from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix
 
 FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt"
 CHANGELOG_IN_PATH = "debian/changelog.in"
@@ -34,8 +34,6 @@ SET(VERSION_STRING {string})
 # end of autochange
 """
 
-git = Git()
-
 
 class ClickHouseVersion:
     """Immutable version class. On update returns a new instance"""
@@ -46,7 +44,7 @@ class ClickHouseVersion:
         minor: Union[int, str],
         patch: Union[int, str],
         revision: Union[int, str],
-        git: Git,
+        git: Optional[Git],
         tweak: str = None,
     ):
         self._major = int(major)
@@ -54,9 +52,11 @@ class ClickHouseVersion:
         self._patch = int(patch)
         self._revision = int(revision)
         self._git = git
-        self._tweak = None
+        self._tweak = TWEAK
         if tweak is not None:
             self._tweak = int(tweak)
+        elif self._git is not None:
+            self._tweak = self._git.tweak
         self._describe = ""
 
     def update(self, part: str) -> "ClickHouseVersion":
@@ -91,7 +91,7 @@ class ClickHouseVersion:
 
     @property
     def tweak(self) -> int:
-        return self._tweak or self._git.tweak
+        return self._tweak
 
     @property
     def revision(self) -> int:
@@ -99,7 +99,9 @@ class ClickHouseVersion:
 
     @property
     def githash(self) -> str:
-        return self._git.sha
+        if self._git is not None:
+            return self._git.sha
+        return "0000000000000000000000000000000000000000"
 
     @property
     def describe(self):
@@ -171,7 +173,7 @@ def validate_version(version: str):
 
 
 def get_abs_path(path: str) -> str:
-    return p.abspath(p.join(git.root, path))
+    return p.abspath(p.join(git_runner.cwd, path))
 
 
 def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS:
@@ -197,6 +199,7 @@ def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS:
 
 def get_version_from_repo(
     versions_path: str = FILE_WITH_VERSION_PATH,
+    git: Optional[Git] = None,
 ) -> ClickHouseVersion:
     versions = read_versions(versions_path)
     return ClickHouseVersion(
@@ -208,14 +211,16 @@ def get_version_from_repo(
     )
 
 
-def get_version_from_string(version: str) -> ClickHouseVersion:
+def get_version_from_string(
+    version: str, git: Optional[Git] = None
+) -> ClickHouseVersion:
     validate_version(version)
     parts = version.split(".")
     return ClickHouseVersion(parts[0], parts[1], parts[2], -1, git, parts[3])
 
 
 def get_version_from_tag(tag: str) -> ClickHouseVersion:
-    git.check_tag(tag)
+    Git.check_tag(tag)
     tag = tag[1:].split("-")[0]
     return get_version_from_string(tag)
 
@@ -236,7 +241,7 @@ def version_arg(version: str) -> ClickHouseVersion:
 
 def get_tagged_versions() -> List[ClickHouseVersion]:
     versions = []
-    for tag in git.get_tags():
+    for tag in get_tags():
         try:
             version = get_version_from_tag(tag)
             versions.append(version)
@@ -261,19 +266,20 @@ def update_contributors(
 ):
     # Check if we have shallow checkout by comparing number of lines
     # '--is-shallow-repository' is in git since 2.15, 2017-10-30
-    if git.run("git rev-parse --is-shallow-repository") == "true" and not force:
+    if git_runner.run("git rev-parse --is-shallow-repository") == "true" and not force:
         logging.warning("The repository is shallow, refusing to update contributors")
         if raise_error:
             raise RuntimeError("update_contributors executed on a shallow repository")
         return
 
-    contributors = git.run("git shortlog HEAD --summary")
+    # format: "  1016  Alexey Arno"
+    shortlog = git_runner.run("git shortlog HEAD --summary")
     contributors = sorted(
-        [c.split(maxsplit=1)[-1].replace('"', r"\"") for c in contributors.split("\n")],
+        [c.split(maxsplit=1)[-1].replace('"', r"\"") for c in shortlog.split("\n")],
     )
     contributors = [f'    "{c}",' for c in contributors]
 
-    executer = p.relpath(p.realpath(__file__), git.root)
+    executer = p.relpath(p.realpath(__file__), git_runner.cwd)
     content = CONTRIBUTORS_TEMPLATE.format(
         executer=executer, contributors="\n".join(contributors)
     )