mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-14 11:33:46 +00:00
451 lines
15 KiB
Python
Executable File
451 lines
15 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# In our CI this script runs in style-test containers
|
|
|
|
import argparse
|
|
import logging
|
|
import os.path as p
|
|
import os
|
|
import re
|
|
from datetime import date, datetime, timedelta
|
|
from queue import Empty, Queue
|
|
from subprocess import CalledProcessError, DEVNULL
|
|
from threading import Thread
|
|
from time import sleep
|
|
from typing import Dict, List, Optional, TextIO
|
|
|
|
from fuzzywuzzy.fuzz import ratio # type: ignore
|
|
from github import Github
|
|
from github.GithubException import RateLimitExceededException, UnknownObjectException
|
|
from github.NamedUser import NamedUser
|
|
from github.Issue import Issue
|
|
from github.PullRequest import PullRequest
|
|
from github.Repository import Repository
|
|
from git_helper import is_shallow, git_runner as runner
|
|
|
|
# This array gives the preferred category order, and is also used to
|
|
# normalize category names.
|
|
# Categories are used in .github/PULL_REQUEST_TEMPLATE.md, keep comments there
|
|
# updated accordingly
|
|
categories_preferred_order = (
|
|
"Backward Incompatible Change",
|
|
"New Feature",
|
|
"Performance Improvement",
|
|
"Improvement",
|
|
"Bug Fix",
|
|
"Build/Testing/Packaging Improvement",
|
|
"Other",
|
|
)
|
|
|
|
FROM_REF = ""
|
|
TO_REF = ""
|
|
SHA_IN_CHANGELOG = [] # type: List[str]
|
|
GitHub = Github()
|
|
CACHE_PATH = p.join(p.dirname(p.realpath(__file__)), "gh_cache")
|
|
|
|
|
|
class Description:
|
|
def __init__(
|
|
self, number: int, user: NamedUser, html_url: str, entry: str, category: str
|
|
):
|
|
self.number = number
|
|
self.html_url = html_url
|
|
self.user = user
|
|
self.entry = entry
|
|
self.category = category
|
|
|
|
@property
|
|
def formatted_entry(self) -> str:
|
|
# Substitute issue links.
|
|
# 1) issue number w/o markdown link
|
|
entry = re.sub(
|
|
r"([^[])#([0-9]{4,})",
|
|
r"\1[#\2](https://github.com/ClickHouse/ClickHouse/issues/\2)",
|
|
self.entry,
|
|
)
|
|
# 2) issue URL w/o markdown link
|
|
entry = re.sub(
|
|
r"([^(])https://github.com/ClickHouse/ClickHouse/issues/([0-9]{4,})",
|
|
r"\1[#\2](https://github.com/ClickHouse/ClickHouse/issues/\2)",
|
|
entry,
|
|
)
|
|
# It's possible that we face a secondary rate limit.
|
|
# In this case we should sleep until we get it
|
|
while True:
|
|
try:
|
|
user_name = self.user.name if self.user.name else self.user.login
|
|
break
|
|
except UnknownObjectException:
|
|
user_name = self.user.login
|
|
break
|
|
except RateLimitExceededException:
|
|
sleep_on_rate_limit()
|
|
return (
|
|
f"* {entry} [#{self.number}]({self.html_url}) "
|
|
f"([{user_name}]({self.user.html_url}))."
|
|
)
|
|
|
|
# Sort PR descriptions by numbers
|
|
def __eq__(self, other) -> bool:
|
|
if not isinstance(self, type(other)):
|
|
return NotImplemented
|
|
return self.number == other.number
|
|
|
|
def __lt__(self, other: "Description") -> bool:
|
|
return self.number < other.number
|
|
|
|
|
|
class Worker(Thread):
|
|
def __init__(self, request_queue: Queue, repo: Repository):
|
|
Thread.__init__(self)
|
|
self.queue = request_queue
|
|
self.repo = repo
|
|
self.response = [] # type: List[Description]
|
|
|
|
def run(self):
|
|
while not self.queue.empty():
|
|
try:
|
|
issue = self.queue.get() # type: Issue
|
|
except Empty:
|
|
break # possible race condition, just continue
|
|
api_pr = get_pull_cached(self.repo, issue.number, issue.updated_at)
|
|
in_changelog = False
|
|
merge_commit = api_pr.merge_commit_sha
|
|
try:
|
|
runner.run(f"git rev-parse '{merge_commit}'")
|
|
except CalledProcessError:
|
|
# It's possible that commit not in the repo, just continue
|
|
logging.info("PR %s does not belong to the repo", api_pr.number)
|
|
continue
|
|
|
|
in_changelog = merge_commit in SHA_IN_CHANGELOG
|
|
if in_changelog:
|
|
desc = generate_description(api_pr, self.repo)
|
|
if desc is not None:
|
|
self.response.append(desc)
|
|
|
|
self.queue.task_done()
|
|
|
|
|
|
def sleep_on_rate_limit(time: int = 20):
|
|
logging.warning("Faced rate limit, sleeping %s", time)
|
|
sleep(time)
|
|
|
|
|
|
def get_pull_cached(
|
|
repo: Repository, number: int, updated_at: Optional[datetime] = None
|
|
) -> PullRequest:
|
|
pr_cache_file = p.join(CACHE_PATH, f"{number}.pickle")
|
|
if updated_at is None:
|
|
updated_at = datetime.now() - timedelta(hours=-1)
|
|
|
|
if p.isfile(pr_cache_file):
|
|
cache_updated = datetime.fromtimestamp(p.getmtime(pr_cache_file))
|
|
if cache_updated > updated_at:
|
|
with open(pr_cache_file, "rb") as prfd:
|
|
return GitHub.load(prfd) # type: ignore
|
|
while True:
|
|
try:
|
|
pr = repo.get_pull(number)
|
|
break
|
|
except RateLimitExceededException:
|
|
sleep_on_rate_limit()
|
|
with open(pr_cache_file, "wb") as prfd:
|
|
GitHub.dump(pr, prfd) # type: ignore
|
|
return pr
|
|
|
|
|
|
def get_descriptions(
|
|
repo: Repository, issues: List[Issue], jobs: int
|
|
) -> Dict[str, List[Description]]:
|
|
workers = [] # type: List[Worker]
|
|
queue = Queue() # type: Queue[Issue]
|
|
for issue in issues:
|
|
queue.put(issue)
|
|
for _ in range(jobs):
|
|
worker = Worker(queue, repo)
|
|
worker.start()
|
|
workers.append(worker)
|
|
|
|
descriptions = {} # type: Dict[str, List[Description]]
|
|
for worker in workers:
|
|
worker.join()
|
|
for desc in worker.response:
|
|
if desc.category not in descriptions:
|
|
descriptions[desc.category] = []
|
|
descriptions[desc.category].append(desc)
|
|
|
|
for descs in descriptions.values():
|
|
descs.sort()
|
|
|
|
return descriptions
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
description="Generate a changelog in MD format between given tags. "
|
|
"It fetches all tags and unshallow the git repositore automatically",
|
|
)
|
|
parser.add_argument(
|
|
"-v",
|
|
"--verbose",
|
|
action="count",
|
|
default=0,
|
|
help="set the script verbosity, could be used multiple",
|
|
)
|
|
parser.add_argument(
|
|
"--output",
|
|
type=argparse.FileType("w"),
|
|
default="-",
|
|
help="output file for changelog",
|
|
)
|
|
parser.add_argument(
|
|
"--repo",
|
|
default="ClickHouse/ClickHouse",
|
|
help="a repository to query for pull-requests from GitHub",
|
|
)
|
|
parser.add_argument(
|
|
"--jobs",
|
|
type=int,
|
|
default=10,
|
|
help="number of jobs to get pull-requests info from GitHub API",
|
|
)
|
|
parser.add_argument(
|
|
"--gh-user-or-token",
|
|
help="user name or GH token to authenticate",
|
|
)
|
|
parser.add_argument(
|
|
"--gh-password",
|
|
help="a password that should be used when user is given",
|
|
)
|
|
parser.add_argument(
|
|
"--with-testing-tags",
|
|
action="store_true",
|
|
help="by default '*-testing' tags are ignored, this argument enables them too",
|
|
)
|
|
parser.add_argument(
|
|
"--from",
|
|
dest="from_ref",
|
|
help="git ref for a starting point of changelog, by default is calculated "
|
|
"automatically to match a previous tag in history",
|
|
)
|
|
parser.add_argument(
|
|
"to_ref",
|
|
metavar="TO_REF",
|
|
help="git ref for the changelog end",
|
|
)
|
|
args = parser.parse_args()
|
|
return args
|
|
|
|
|
|
# This function mirrors the PR description checks in ClickhousePullRequestTrigger.
|
|
# Returns False if the PR should not be mentioned changelog.
|
|
def generate_description(item: PullRequest, repo: Repository) -> Optional[Description]:
|
|
backport_number = item.number
|
|
if item.head.ref.startswith("backport/"):
|
|
branch_parts = item.head.ref.split("/")
|
|
if len(branch_parts) == 3:
|
|
try:
|
|
item = get_pull_cached(repo, int(branch_parts[-1]))
|
|
except Exception as e:
|
|
logging.warning("unable to get backpoted PR, exception: %s", e)
|
|
else:
|
|
logging.warning(
|
|
"The branch %s doesn't match backport template, using PR %s as is",
|
|
item.head.ref,
|
|
item.number,
|
|
)
|
|
description = item.body
|
|
# Don't skip empty lines because they delimit parts of description
|
|
lines = [x.strip() for x in (description.split("\n") if description else [])]
|
|
lines = [re.sub(r"\s+", " ", ln) for ln in lines]
|
|
|
|
category = ""
|
|
entry = ""
|
|
|
|
if lines:
|
|
i = 0
|
|
while i < len(lines):
|
|
if re.match(r"(?i)^[#>*_ ]*change\s*log\s*category", lines[i]):
|
|
i += 1
|
|
if i >= len(lines):
|
|
break
|
|
# Can have one empty line between header and the category itself.
|
|
# Filter it out.
|
|
if not lines[i]:
|
|
i += 1
|
|
if i >= len(lines):
|
|
break
|
|
category = re.sub(r"^[-*\s]*", "", lines[i])
|
|
i += 1
|
|
elif re.match(
|
|
r"(?i)^[#>*_ ]*(short\s*description|change\s*log\s*entry)", lines[i]
|
|
):
|
|
i += 1
|
|
# Can have one empty line between header and the entry itself.
|
|
# Filter it out.
|
|
if i < len(lines) and not lines[i]:
|
|
i += 1
|
|
# All following lines until empty one are the changelog entry.
|
|
entry_lines = []
|
|
while i < len(lines) and lines[i]:
|
|
entry_lines.append(lines[i])
|
|
i += 1
|
|
entry = " ".join(entry_lines)
|
|
else:
|
|
i += 1
|
|
|
|
if not category:
|
|
# Shouldn't happen, because description check in CI should catch such PRs.
|
|
# Fall through, so that it shows up in output and the user can fix it.
|
|
category = "NO CL CATEGORY"
|
|
|
|
# Filter out the PR categories that are not for changelog.
|
|
if re.match(
|
|
r"(?i)((non|in|not|un)[-\s]*significant)|(not[ ]*for[ ]*changelog)",
|
|
category,
|
|
):
|
|
category = "NOT FOR CHANGELOG / INSIGNIFICANT"
|
|
return Description(item.number, item.user, item.html_url, item.title, category)
|
|
|
|
# Filter out documentations changelog
|
|
if re.match(
|
|
r"(?i)doc",
|
|
category,
|
|
):
|
|
return None
|
|
|
|
if backport_number != item.number:
|
|
entry = f"Backported in #{backport_number}: {entry}"
|
|
|
|
if not entry:
|
|
# Shouldn't happen, because description check in CI should catch such PRs.
|
|
category = "NO CL ENTRY"
|
|
entry = "NO CL ENTRY: '" + item.title + "'"
|
|
|
|
entry = entry.strip()
|
|
if entry[-1] != ".":
|
|
entry += "."
|
|
|
|
for c in categories_preferred_order:
|
|
if ratio(category.lower(), c.lower()) >= 90:
|
|
category = c
|
|
break
|
|
|
|
return Description(item.number, item.user, item.html_url, entry, category)
|
|
|
|
|
|
def write_changelog(fd: TextIO, descriptions: Dict[str, List[Description]]):
|
|
year = date.today().year
|
|
fd.write(
|
|
f"---\nsidebar_position: 1\nsidebar_label: {year}\n---\n\n# {year} Changelog\n\n"
|
|
f"### ClickHouse release {TO_REF} FIXME as compared to {FROM_REF}\n\n"
|
|
)
|
|
|
|
seen_categories = [] # type: List[str]
|
|
for category in categories_preferred_order:
|
|
if category in descriptions:
|
|
seen_categories.append(category)
|
|
fd.write(f"#### {category}\n")
|
|
for desc in descriptions[category]:
|
|
fd.write(f"{desc.formatted_entry}\n")
|
|
|
|
fd.write("\n")
|
|
|
|
for category in sorted(descriptions):
|
|
if category not in seen_categories:
|
|
fd.write(f"#### {category}\n\n")
|
|
for desc in descriptions[category]:
|
|
fd.write(f"{desc.formatted_entry}\n")
|
|
|
|
fd.write("\n")
|
|
|
|
|
|
def check_refs(from_ref: Optional[str], to_ref: str, with_testing_tags: bool):
|
|
global FROM_REF, TO_REF
|
|
TO_REF = to_ref
|
|
|
|
# Check TO_REF
|
|
runner.run(f"git rev-parse {TO_REF}")
|
|
|
|
# Check from_ref
|
|
if from_ref is None:
|
|
# Get all tags pointing to TO_REF
|
|
tags = runner.run(f"git tag --points-at '{TO_REF}^{{}}'").split("\n")
|
|
logging.info("All tags pointing to %s:\n%s", TO_REF, tags)
|
|
if not with_testing_tags:
|
|
tags.append("*-testing")
|
|
exclude = " ".join([f"--exclude='{tag}'" for tag in tags])
|
|
cmd = f"git describe --abbrev=0 --tags {exclude} '{TO_REF}'"
|
|
FROM_REF = runner.run(cmd)
|
|
else:
|
|
runner.run(f"git rev-parse {FROM_REF}")
|
|
FROM_REF = from_ref
|
|
|
|
|
|
def set_sha_in_changelog():
|
|
global SHA_IN_CHANGELOG
|
|
SHA_IN_CHANGELOG = runner.run(
|
|
f"git log --format=format:%H {FROM_REF}..{TO_REF}"
|
|
).split("\n")
|
|
|
|
|
|
def main():
|
|
log_levels = [logging.CRITICAL, logging.WARN, logging.INFO, logging.DEBUG]
|
|
args = parse_args()
|
|
logging.basicConfig(
|
|
format="%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d]:\n%(message)s",
|
|
level=log_levels[min(args.verbose, 3)],
|
|
)
|
|
# Create a cache directory
|
|
if not p.isdir(CACHE_PATH):
|
|
os.mkdir(CACHE_PATH, 0o700)
|
|
|
|
# Get the full repo
|
|
if is_shallow():
|
|
logging.info("Unshallow repository")
|
|
runner.run("git fetch --unshallow", stderr=DEVNULL)
|
|
logging.info("Fetching all tags")
|
|
runner.run("git fetch --tags", stderr=DEVNULL)
|
|
|
|
check_refs(args.from_ref, args.to_ref, args.with_testing_tags)
|
|
set_sha_in_changelog()
|
|
|
|
logging.info("Using %s..%s as changelog interval", FROM_REF, TO_REF)
|
|
|
|
# Get starting and ending dates for gathering PRs
|
|
# Add one day after and before to mitigate TZ possible issues
|
|
# `tag^{}` format gives commit ref when we have annotated tags
|
|
# format %cs gives a committer date, works better for cherry-picked commits
|
|
from_date = runner.run(f"git log -1 --format=format:%cs '{FROM_REF}^{{}}'")
|
|
from_date = (date.fromisoformat(from_date) - timedelta(1)).isoformat()
|
|
to_date = runner.run(f"git log -1 --format=format:%cs '{TO_REF}^{{}}'")
|
|
to_date = (date.fromisoformat(to_date) + timedelta(1)).isoformat()
|
|
|
|
# Get all PRs for the given time frame
|
|
global GitHub
|
|
GitHub = Github(
|
|
args.gh_user_or_token, args.gh_password, per_page=100, pool_size=args.jobs
|
|
)
|
|
query = f"type:pr repo:{args.repo} is:merged merged:{from_date}..{to_date}"
|
|
repo = GitHub.get_repo(args.repo)
|
|
api_prs = GitHub.search_issues(query=query, sort="created")
|
|
logging.info("Found %s PRs for the query: '%s'", api_prs.totalCount, query)
|
|
|
|
issues = [] # type: List[Issue]
|
|
while True:
|
|
try:
|
|
for issue in api_prs:
|
|
issues.append(issue)
|
|
break
|
|
except RateLimitExceededException:
|
|
sleep_on_rate_limit()
|
|
|
|
descriptions = get_descriptions(repo, issues, args.jobs)
|
|
|
|
write_changelog(args.output, descriptions)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|