ClickHouse/tests/ci/pr_info.py

200 lines
8.0 KiB
Python
Raw Normal View History

2021-09-15 12:59:39 +00:00
#!/usr/bin/env python3
import json
2021-10-29 13:57:47 +00:00
import os
2021-09-15 17:48:39 +00:00
import urllib
2021-10-29 13:57:47 +00:00
import requests
2021-09-15 16:32:17 +00:00
from unidiff import PatchSet
2021-11-26 14:00:09 +00:00
from env_helper import GITHUB_REPOSITORY, GITHUB_SERVER_URL, GITHUB_RUN_ID, GITHUB_EVENT_PATH
2021-09-15 12:59:39 +00:00
2021-11-26 14:00:09 +00:00
DIFF_IN_DOCUMENTATION_EXT = [".html", ".md", ".yml", ".txt", ".css", ".js", ".xml", ".ico", ".conf", ".svg", ".png",
".jpg", ".py", ".sh", ".json"]
2021-10-29 09:58:25 +00:00
2021-11-22 09:56:13 +00:00
def get_pr_for_commit(sha, ref):
2021-11-26 14:00:09 +00:00
try_get_pr_url = f"https://api.github.com/repos/{GITHUB_REPOSITORY}/commits/{sha}/pulls"
2021-11-22 09:39:45 +00:00
try:
response = requests.get(try_get_pr_url)
response.raise_for_status()
data = response.json()
if len(data) > 1:
print("Got more than one pr for commit", sha)
for pr in data:
2021-11-22 09:56:13 +00:00
# refs for pushes looks like refs/head/XX
# refs for RPs looks like XX
if pr['head']['ref'] in ref:
2021-11-22 09:39:45 +00:00
return pr
2021-11-26 14:00:09 +00:00
print("Cannot find PR with required ref", ref, "returning first one")
2021-11-22 09:39:45 +00:00
first_pr = data[0]
return first_pr
except Exception as ex:
print("Cannot fetch PR info from commit", ex)
return None
2021-09-15 12:59:39 +00:00
class PRInfo:
2021-12-22 07:54:50 +00:00
def __init__(self, github_event=None, need_orgs=False, need_changed_files=False, labels_from_api=False):
2021-11-26 14:00:09 +00:00
if not github_event:
if GITHUB_EVENT_PATH:
with open(GITHUB_EVENT_PATH, 'r', encoding='utf-8') as event_file:
github_event = json.load(event_file)
else:
github_event = {'commits': 1, 'after': 'HEAD', 'ref': None}
self.event = github_event
self.changed_files = set([])
if 'pull_request' in github_event: # pull request and other similar events
2021-10-29 15:01:29 +00:00
self.number = github_event['number']
if 'after' in github_event:
self.sha = github_event['after']
else:
self.sha = github_event['pull_request']['head']['sha']
2021-11-26 14:00:09 +00:00
repo_prefix = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}"
self.task_url = f"{repo_prefix}/actions/runs/{GITHUB_RUN_ID or '0'}"
2021-11-19 14:47:04 +00:00
2021-11-26 14:00:09 +00:00
self.repo_full_name = GITHUB_REPOSITORY
2021-11-19 14:47:04 +00:00
self.commit_html_url = f"{repo_prefix}/commits/{self.sha}"
self.pr_html_url = f"{repo_prefix}/pull/{self.number}"
self.base_ref = github_event['pull_request']['base']['ref']
self.base_name = github_event['pull_request']['base']['repo']['full_name']
self.head_ref = github_event['pull_request']['head']['ref']
2021-11-19 15:09:18 +00:00
self.head_name = github_event['pull_request']['head']['repo']['full_name']
2021-11-19 14:47:04 +00:00
2021-12-22 07:54:50 +00:00
if labels_from_api:
2021-12-22 08:15:31 +00:00
response = requests.get(f"https://api.github.com/repos/{GITHUB_REPOSITORY}/issues/{self.number}/labels")
2021-12-22 07:54:50 +00:00
self.labels = {l['name'] for l in response.json()}
2021-12-22 08:13:04 +00:00
else:
self.labels = {l['name'] for l in github_event['pull_request']['labels']}
2021-12-22 07:54:50 +00:00
2021-10-29 15:01:29 +00:00
self.user_login = github_event['pull_request']['user']['login']
self.user_orgs = set([])
if need_orgs:
user_orgs_response = requests.get(github_event['pull_request']['user']['organizations_url'])
if user_orgs_response.ok:
response_json = user_orgs_response.json()
self.user_orgs = set(org['id'] for org in response_json)
2021-11-26 14:00:09 +00:00
self.diff_url = github_event['pull_request']['diff_url']
2021-12-15 13:35:53 +00:00
elif 'commits' in github_event:
2021-09-15 13:45:44 +00:00
self.sha = github_event['after']
2021-11-22 09:56:13 +00:00
pull_request = get_pr_for_commit(self.sha, github_event['ref'])
2021-11-26 14:00:09 +00:00
repo_prefix = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}"
self.task_url = f"{repo_prefix}/actions/runs/{GITHUB_RUN_ID or '0'}"
2021-11-19 14:47:04 +00:00
self.commit_html_url = f"{repo_prefix}/commits/{self.sha}"
2021-11-26 14:00:09 +00:00
self.repo_full_name = GITHUB_REPOSITORY
if pull_request is None or pull_request['state'] == 'closed': # it's merged PR to master
2021-11-22 09:39:45 +00:00
self.number = 0
self.labels = {}
self.pr_html_url = f"{repo_prefix}/commits/master"
self.base_ref = "master"
self.base_name = self.repo_full_name
self.head_ref = "master"
self.head_name = self.repo_full_name
2021-11-26 14:00:09 +00:00
self.diff_url = \
f"https://api.github.com/repos/{GITHUB_REPOSITORY}/compare/{github_event['before']}...{self.sha}"
2021-11-22 09:39:45 +00:00
else:
self.number = pull_request['number']
2021-12-22 07:54:50 +00:00
if labels_from_api:
2021-12-22 08:15:31 +00:00
response = requests.get(f"https://api.github.com/repos/{GITHUB_REPOSITORY}/issues/{self.number}/labels")
2021-12-22 07:54:50 +00:00
self.labels = {l['name'] for l in response.json()}
2021-12-22 08:13:04 +00:00
else:
self.labels = {l['name'] for l in pull_request['labels']}
2021-11-22 09:39:45 +00:00
self.base_ref = pull_request['base']['ref']
self.base_name = pull_request['base']['repo']['full_name']
self.head_ref = pull_request['head']['ref']
self.head_name = pull_request['head']['repo']['full_name']
self.pr_html_url = pull_request['html_url']
2021-11-26 14:00:09 +00:00
if 'pr-backport' in self.labels:
self.diff_url = f"https://github.com/{GITHUB_REPOSITORY}/compare/master...{self.head_ref}.diff"
2021-10-29 15:01:29 +00:00
else:
2021-11-26 14:00:09 +00:00
self.diff_url = pull_request['diff_url']
2021-10-31 18:08:38 +00:00
else:
raise Exception("Cannot detect type of event")
2021-11-26 14:00:09 +00:00
if need_changed_files:
self.fetch_changed_files()
2021-10-31 18:08:38 +00:00
2021-11-26 14:00:09 +00:00
def fetch_changed_files(self):
if 'commits' in self.event and self.number == 0:
response = requests.get(self.diff_url)
response.raise_for_status()
diff = response.json()
if 'files' in diff:
self.changed_files = [f['filename'] for f in diff['files']]
else:
diff = urllib.request.urlopen(self.diff_url)
diff_object = PatchSet(diff, diff.headers.get_charsets()[0])
self.changed_files = {f.path for f in diff_object}
2021-09-15 16:32:17 +00:00
2021-09-15 13:56:03 +00:00
def get_dict(self):
return {
'sha': self.sha,
'number': self.number,
'labels': self.labels,
'user_login': self.user_login,
'user_orgs': self.user_orgs,
}
2021-10-21 15:32:15 +00:00
2021-10-29 09:58:25 +00:00
def has_changes_in_documentation(self):
# If the list wasn't built yet the best we can do is to
# assume that there were changes.
if self.changed_files is None or not self.changed_files:
return True
for f in self.changed_files:
_, ext = os.path.splitext(f)
2021-11-25 15:23:00 +00:00
path_in_docs = 'docs' in f
path_in_website = 'website' in f
2021-11-30 07:42:02 +00:00
if (ext in DIFF_IN_DOCUMENTATION_EXT and (path_in_docs or path_in_website)) or 'docker/docs' in f:
2021-10-29 09:58:25 +00:00
return True
return False
2021-12-03 08:33:16 +00:00
def can_skip_builds_and_use_version_from_master(self):
if 'force tests' in self.labels:
return False
if self.changed_files is None or not self.changed_files:
return False
for f in self.changed_files:
if (not f.startswith('tests/queries')
or not f.startswith('tests/integration')
or not f.startswith('tests/performance')):
return False
return True
def can_skip_integration_tests(self):
if 'force tests' in self.labels:
return False
if self.changed_files is None or not self.changed_files:
return False
for f in self.changed_files:
if not f.startswith('tests/queries') or not f.startswith('tests/performance'):
return False
return True
def can_skip_functional_tests(self):
if 'force tests' in self.labels:
return False
if self.changed_files is None or not self.changed_files:
return False
for f in self.changed_files:
if not f.startswith('tests/integration') or not f.startswith('tests/performance'):
return False
return True
2021-10-21 15:32:15 +00:00
class FakePRInfo:
def __init__(self):
self.number = 11111
self.sha = "xxxxxxxxxxxxxxxxxx"