thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adjust git_helper and version_helper for shallow clone

Browse Source
This commit is contained in:
Mikhail f. Shiryaev 2022-04-12 18:39:02 +02:00
parent 9abd28ebc9
commit c65c832edb
No known key found for this signature in database
GPG Key ID: 4B02ED204C7D93F4
5 changed files with 100 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
tests/ci/build_check.py
View File

@ -14,6 +14,7 @@ from s3_helper import S3Helper
from pr_info import PRInfo
from version_helper import (
ClickHouseVersion,
Git,
get_version_from_repo,
update_version_local,
)
@ -209,7 +210,7 @@ def main():
s3_helper = S3Helper("https://s3.amazonaws.com")
version = get_version_from_repo()
version = get_version_from_repo(git=Git(True))
release_or_pr = get_release_or_pr(pr_info, build_config, version)
s3_path_prefix = "/".join((release_or_pr, pr_info.sha, build_name))

71
tests/ci/git_helper.py
View File

@ -13,6 +13,9 @@ TAG_REGEXP = (
)
SHA_REGEXP = r"\A([0-9]|[a-f]){40}\Z"
CWD = p.dirname(p.realpath(__file__))
TWEAK = 1
# Py 3.8 removeprefix and removesuffix
def removeprefix(string: str, prefix: str):
@ -46,8 +49,8 @@ def release_branch(name: str):
class Runner:
"""lightweight check_output wrapper with stripping last NEW_LINE"""
def __init__(self, cwd: str = p.dirname(p.realpath(__file__))):
self.cwd = cwd
def __init__(self, cwd: str = CWD):
self._cwd = cwd
def run(self, cmd: str, cwd: Optional[str] = None) -> str:
if cwd is None:
@ -56,22 +59,47 @@ class Runner:
cmd, shell=True, cwd=cwd, encoding="utf-8"
).strip()
@property
def cwd(self) -> str:
return self._cwd
@cwd.setter
def cwd(self, value: str):
# Set _cwd only once, then set it to readonly
if self._cwd != CWD:
return
self._cwd = value
git_runner = Runner()
# Set cwd to abs path of git root
git_runner.cwd = p.relpath(
p.join(git_runner.cwd, git_runner.run("git rev-parse --show-cdup"))
)
def get_tags() -> List[str]:
if git_runner.run("git rev-parse --is-shallow-repository") == "true":
raise RuntimeError("attempt to run on a shallow repository")
return git_runner.run("git tag").split()
class Git:
"""A small wrapper around subprocess to invoke git commands"""
def __init__(self):
runner = Runner()
rel_root = runner.run("git rev-parse --show-cdup")
self.root = p.realpath(p.join(runner.cwd, rel_root))
self._tag_pattern = re.compile(TAG_REGEXP)
runner.cwd = self.root
self.run = runner.run
_tag_pattern = re.compile(TAG_REGEXP)
def __init__(self, ignore_no_tags: bool = False):
self.root = git_runner.cwd
self._ignore_no_tags = ignore_no_tags
self.run = git_runner.run
self.latest_tag = ""
self.new_tag = ""
self.new_branch = ""
self.branch = ""
self.sha = ""
self.sha_short = ""
self.description = ""
self.description = "shallow-checkout"
self.commits_since_tag = 0
self.update()
@ -82,6 +110,19 @@ class Git:
self.sha_short = self.sha[:11]
# The following command shows the most recent tag in a graph
# Format should match TAG_REGEXP
if (
self._ignore_no_tags
and self.run("git rev-parse --is-shallow-repository") == "true"
):
try:
self._update_tags()
except subprocess.CalledProcessError:
pass
return
self._update_tags()
def _update_tags(self):
self.latest_tag = self.run("git describe --tags --abbrev=0")
# Format should be: {latest_tag}-{commits_since_tag}-g{sha_short}
self.description = self.run("git describe --tags --long")
@ -89,10 +130,11 @@ class Git:
self.run(f"git rev-list {self.latest_tag}..HEAD --count")
)
def check_tag(self, value: str):
@staticmethod
def check_tag(value: str):
if value == "":
return
if not self._tag_pattern.match(value):
if not Git._tag_pattern.match(value):
raise ValueError(f"last tag {value} doesn't match the pattern")
@property
@ -118,10 +160,7 @@ class Git:
if not self.latest_tag.endswith("-testing"):
# When we are on the tag, we still need to have tweak=1 to not
# break cmake with versions like 12.13.14.0
return self.commits_since_tag or 1
return self.commits_since_tag or TWEAK
version = self.latest_tag.split("-", maxsplit=1)[0]
return int(version.split(".")[-1]) + self.commits_since_tag
def get_tags(self) -> List[str]:
return self.run("git tag").split()

15
tests/ci/git_test.py
View File

@ -4,7 +4,7 @@ from unittest.mock import patch
import os.path as p
import unittest
from git_helper import Git, Runner
from git_helper import Git, Runner, CWD
class TestRunner(unittest.TestCase):
@ -19,6 +19,18 @@ class TestRunner(unittest.TestCase):
output = runner.run("echo 1")
self.assertEqual(output, "1")
def test_one_time_writeable_cwd(self):
runner = Runner()
self.assertEqual(runner.cwd, CWD)
runner.cwd = "/bin"
self.assertEqual(runner.cwd, "/bin")
runner.cwd = "/"
self.assertEqual(runner.cwd, "/bin")
runner = Runner("/")
self.assertEqual(runner.cwd, "/")
runner.cwd = "/bin"
self.assertEqual(runner.cwd, "/")
class TestGit(unittest.TestCase):
def setUp(self):
@ -31,6 +43,7 @@ class TestGit(unittest.TestCase):
self.addCleanup(update_patcher.stop)
self.git = Git()
update_mock.assert_called_once()
self.git.run("test")
self.run_mock.assert_called_once()
self.git.new_branch = "NEW_BRANCH_NAME"
self.git.new_tag = "v21.12.333.22222-stable"

10
tests/ci/release.py
View File

@ -10,6 +10,7 @@ from git_helper import commit, release_branch
from version_helper import (
FILE_WITH_VERSION_PATH,
ClickHouseVersion,
Git,
VersionType,
get_abs_path,
get_version_from_repo,
@ -17,6 +18,9 @@ from version_helper import (
)
git = Git()
class Repo:
VALID = ("ssh", "https", "origin")
@ -53,8 +57,8 @@ class Release:
self._release_commit = ""
self.release_commit = release_commit
self.release_type = release_type
self._version = get_version_from_repo()
self._git = self._version._git
self._git = git
self._version = get_version_from_repo(git=self._git)
self._release_branch = ""
self._rollback_stack = [] # type: List[str]
@ -75,7 +79,7 @@ class Release:
def read_version(self):
self._git.update()
self.version = get_version_from_repo()
self.version = get_version_from_repo(git=self._git)
def check_prerequisites(self):
"""

38
tests/ci/version_helper.py
View File

@ -2,9 +2,9 @@
import logging
import os.path as p
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, ArgumentTypeError
from typing import Dict, List, Tuple, Union
from typing import Dict, List, Optional, Tuple, Union
from git_helper import Git, removeprefix
from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix
FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt"
CHANGELOG_IN_PATH = "debian/changelog.in"
@ -34,8 +34,6 @@ SET(VERSION_STRING {string})
# end of autochange
"""
git = Git()
class ClickHouseVersion:
"""Immutable version class. On update returns a new instance"""
@ -46,7 +44,7 @@ class ClickHouseVersion:
minor: Union[int, str],
patch: Union[int, str],
revision: Union[int, str],
git: Git,
git: Optional[Git],
tweak: str = None,
):
self._major = int(major)
@ -54,9 +52,11 @@ class ClickHouseVersion:
self._patch = int(patch)
self._revision = int(revision)
self._git = git
self._tweak = None
self._tweak = TWEAK
if tweak is not None:
self._tweak = int(tweak)
elif self._git is not None:
self._tweak = self._git.tweak
self._describe = ""
def update(self, part: str) -> "ClickHouseVersion":
@ -91,7 +91,7 @@ class ClickHouseVersion:
@property
def tweak(self) -> int:
return self._tweak or self._git.tweak
return self._tweak
@property
def revision(self) -> int:
@ -99,7 +99,9 @@ class ClickHouseVersion:
@property
def githash(self) -> str:
return self._git.sha
if self._git is not None:
return self._git.sha
return "0000000000000000000000000000000000000000"
@property
def describe(self):
@ -171,7 +173,7 @@ def validate_version(version: str):
def get_abs_path(path: str) -> str:
return p.abspath(p.join(git.root, path))
return p.abspath(p.join(git_runner.cwd, path))
def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS:
@ -197,6 +199,7 @@ def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS:
def get_version_from_repo(
versions_path: str = FILE_WITH_VERSION_PATH,
git: Optional[Git] = None,
) -> ClickHouseVersion:
versions = read_versions(versions_path)
return ClickHouseVersion(
@ -208,14 +211,16 @@ def get_version_from_repo(
)
def get_version_from_string(version: str) -> ClickHouseVersion:
def get_version_from_string(
version: str, git: Optional[Git] = None
) -> ClickHouseVersion:
validate_version(version)
parts = version.split(".")
return ClickHouseVersion(parts[0], parts[1], parts[2], -1, git, parts[3])
def get_version_from_tag(tag: str) -> ClickHouseVersion:
git.check_tag(tag)
Git.check_tag(tag)
tag = tag[1:].split("-")[0]
return get_version_from_string(tag)
@ -236,7 +241,7 @@ def version_arg(version: str) -> ClickHouseVersion:
def get_tagged_versions() -> List[ClickHouseVersion]:
versions = []
for tag in git.get_tags():
for tag in get_tags():
try:
version = get_version_from_tag(tag)
versions.append(version)
@ -261,19 +266,20 @@ def update_contributors(
):
# Check if we have shallow checkout by comparing number of lines
# '--is-shallow-repository' is in git since 2.15, 2017-10-30
if git.run("git rev-parse --is-shallow-repository") == "true" and not force:
if git_runner.run("git rev-parse --is-shallow-repository") == "true" and not force:
logging.warning("The repository is shallow, refusing to update contributors")
if raise_error:
raise RuntimeError("update_contributors executed on a shallow repository")
return
contributors = git.run("git shortlog HEAD --summary")
# format: " 1016 Alexey Arno"
shortlog = git_runner.run("git shortlog HEAD --summary")
contributors = sorted(
[c.split(maxsplit=1)[-1].replace('"', r"\"") for c in contributors.split("\n")],
[c.split(maxsplit=1)[-1].replace('"', r"\"") for c in shortlog.split("\n")],
)
contributors = [f' "{c}",' for c in contributors]
executer = p.relpath(p.realpath(__file__), git.root)
executer = p.relpath(p.realpath(__file__), git_runner.cwd)
content = CONTRIBUTORS_TEMPLATE.format(
executer=executer, contributors="\n".join(contributors)
)