mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 09:32:06 +00:00
Fix run_check.py and dependencies
This commit is contained in:
parent
78e560f78e
commit
9dff6a80ab
@ -5,7 +5,7 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from typing import List, Optional
|
from typing import Any, List, Optional
|
||||||
|
|
||||||
import requests # type: ignore
|
import requests # type: ignore
|
||||||
|
|
||||||
@ -18,7 +18,7 @@ def get_with_retries(
|
|||||||
url: str,
|
url: str,
|
||||||
retries: int = DOWNLOAD_RETRIES_COUNT,
|
retries: int = DOWNLOAD_RETRIES_COUNT,
|
||||||
sleep: int = 3,
|
sleep: int = 3,
|
||||||
**kwargs,
|
**kwargs: Any,
|
||||||
) -> requests.Response:
|
) -> requests.Response:
|
||||||
logging.info(
|
logging.info(
|
||||||
"Getting URL with %i tries and sleep %i in between: %s", retries, sleep, url
|
"Getting URL with %i tries and sleep %i in between: %s", retries, sleep, url
|
||||||
@ -41,18 +41,18 @@ def get_with_retries(
|
|||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
||||||
def get_build_name_for_check(check_name) -> str:
|
def get_build_name_for_check(check_name: str) -> str:
|
||||||
return CI_CONFIG["tests_config"][check_name]["required_build"]
|
return CI_CONFIG["tests_config"][check_name]["required_build"] # type: ignore
|
||||||
|
|
||||||
|
|
||||||
def read_build_urls(build_name, reports_path) -> List[str]:
|
def read_build_urls(build_name: str, reports_path: str) -> List[str]:
|
||||||
for root, _, files in os.walk(reports_path):
|
for root, _, files in os.walk(reports_path):
|
||||||
for f in files:
|
for f in files:
|
||||||
if build_name in f:
|
if build_name in f:
|
||||||
logging.info("Found build report json %s", f)
|
logging.info("Found build report json %s", f)
|
||||||
with open(os.path.join(root, f), "r", encoding="utf-8") as file_handler:
|
with open(os.path.join(root, f), "r", encoding="utf-8") as file_handler:
|
||||||
build_report = json.load(file_handler)
|
build_report = json.load(file_handler)
|
||||||
return build_report["build_urls"]
|
return build_report["build_urls"] # type: ignore
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
@ -17,7 +17,7 @@ RETRY = 5
|
|||||||
CommitStatuses = List[CommitStatus]
|
CommitStatuses = List[CommitStatus]
|
||||||
|
|
||||||
|
|
||||||
def override_status(status: str, check_name: str, invert=False) -> str:
|
def override_status(status: str, check_name: str, invert: bool = False) -> str:
|
||||||
if CI_CONFIG["tests_config"].get(check_name, {}).get("force_tests", False):
|
if CI_CONFIG["tests_config"].get(check_name, {}).get("force_tests", False):
|
||||||
return "success"
|
return "success"
|
||||||
|
|
||||||
@ -45,7 +45,7 @@ def get_commit(gh: Github, commit_sha: str, retry_count: int = RETRY) -> Commit:
|
|||||||
|
|
||||||
def post_commit_status(
|
def post_commit_status(
|
||||||
gh: Github, sha: str, check_name: str, description: str, state: str, report_url: str
|
gh: Github, sha: str, check_name: str, description: str, state: str, report_url: str
|
||||||
):
|
) -> None:
|
||||||
for i in range(RETRY):
|
for i in range(RETRY):
|
||||||
try:
|
try:
|
||||||
commit = get_commit(gh, sha, 1)
|
commit = get_commit(gh, sha, 1)
|
||||||
@ -64,7 +64,7 @@ def post_commit_status(
|
|||||||
|
|
||||||
def post_commit_status_to_file(
|
def post_commit_status_to_file(
|
||||||
file_path: str, description: str, state: str, report_url: str
|
file_path: str, description: str, state: str, report_url: str
|
||||||
):
|
) -> None:
|
||||||
if os.path.exists(file_path):
|
if os.path.exists(file_path):
|
||||||
raise Exception(f'File "{file_path}" already exists!')
|
raise Exception(f'File "{file_path}" already exists!')
|
||||||
with open(file_path, "w", encoding="utf-8") as f:
|
with open(file_path, "w", encoding="utf-8") as f:
|
||||||
@ -88,21 +88,21 @@ def get_commit_filtered_statuses(commit: Commit) -> CommitStatuses:
|
|||||||
return list(filtered.values())
|
return list(filtered.values())
|
||||||
|
|
||||||
|
|
||||||
def remove_labels(gh: Github, pr_info: PRInfo, labels_names: List[str]):
|
def remove_labels(gh: Github, pr_info: PRInfo, labels_names: List[str]) -> None:
|
||||||
repo = gh.get_repo(GITHUB_REPOSITORY)
|
repo = gh.get_repo(GITHUB_REPOSITORY)
|
||||||
pull_request = repo.get_pull(pr_info.number)
|
pull_request = repo.get_pull(pr_info.number)
|
||||||
for label in labels_names:
|
for label in labels_names:
|
||||||
pull_request.remove_from_labels(label)
|
pull_request.remove_from_labels(label)
|
||||||
|
|
||||||
|
|
||||||
def post_labels(gh: Github, pr_info: PRInfo, labels_names: List[str]):
|
def post_labels(gh: Github, pr_info: PRInfo, labels_names: List[str]) -> None:
|
||||||
repo = gh.get_repo(GITHUB_REPOSITORY)
|
repo = gh.get_repo(GITHUB_REPOSITORY)
|
||||||
pull_request = repo.get_pull(pr_info.number)
|
pull_request = repo.get_pull(pr_info.number)
|
||||||
for label in labels_names:
|
for label in labels_names:
|
||||||
pull_request.add_to_labels(label)
|
pull_request.add_to_labels(label)
|
||||||
|
|
||||||
|
|
||||||
def fail_mergeable_check(commit: Commit, description: str):
|
def fail_mergeable_check(commit: Commit, description: str) -> None:
|
||||||
commit.create_status(
|
commit.create_status(
|
||||||
context="Mergeable Check",
|
context="Mergeable Check",
|
||||||
description=description,
|
description=description,
|
||||||
@ -111,7 +111,7 @@ def fail_mergeable_check(commit: Commit, description: str):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def reset_mergeable_check(commit: Commit, description: str = ""):
|
def reset_mergeable_check(commit: Commit, description: str = "") -> None:
|
||||||
commit.create_status(
|
commit.create_status(
|
||||||
context="Mergeable Check",
|
context="Mergeable Check",
|
||||||
description=description,
|
description=description,
|
||||||
@ -120,7 +120,7 @@ def reset_mergeable_check(commit: Commit, description: str = ""):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def update_mergeable_check(gh: Github, pr_info: PRInfo, check_name: str):
|
def update_mergeable_check(gh: Github, pr_info: PRInfo, check_name: str) -> None:
|
||||||
if SKIP_MERGEABLE_CHECK_LABEL in pr_info.labels:
|
if SKIP_MERGEABLE_CHECK_LABEL in pr_info.labels:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
@ -1,8 +1,17 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
import logging
|
import logging
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
import boto3 # type: ignore
|
import boto3 # type: ignore
|
||||||
from github import Github # type: ignore
|
from github import Github
|
||||||
|
from github.AuthenticatedUser import AuthenticatedUser
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Token:
|
||||||
|
user: AuthenticatedUser
|
||||||
|
value: str
|
||||||
|
rest: int
|
||||||
|
|
||||||
|
|
||||||
def get_parameter_from_ssm(name, decrypt=True, client=None):
|
def get_parameter_from_ssm(name, decrypt=True, client=None):
|
||||||
@ -19,7 +28,7 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"):
|
|||||||
]
|
]
|
||||||
)["Parameters"]
|
)["Parameters"]
|
||||||
assert parameters
|
assert parameters
|
||||||
token = {"login": "", "value": "", "rest": 0}
|
token = None
|
||||||
|
|
||||||
for token_name in [p["Name"] for p in parameters]:
|
for token_name in [p["Name"] for p in parameters]:
|
||||||
value = get_parameter_from_ssm(token_name, True, client)
|
value = get_parameter_from_ssm(token_name, True, client)
|
||||||
@ -29,12 +38,15 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"):
|
|||||||
user = gh.get_user()
|
user = gh.get_user()
|
||||||
rest, _ = gh.rate_limiting
|
rest, _ = gh.rate_limiting
|
||||||
logging.info("Get token with %s remaining requests", rest)
|
logging.info("Get token with %s remaining requests", rest)
|
||||||
if token["rest"] < rest:
|
if token is None:
|
||||||
token = {"user": user, "value": value, "rest": rest}
|
token = Token(user, value, rest)
|
||||||
|
continue
|
||||||
|
if token.rest < rest:
|
||||||
|
token.user, token.value, token.rest = user, value, rest
|
||||||
|
|
||||||
assert token["value"]
|
assert token
|
||||||
logging.info(
|
logging.info(
|
||||||
"User %s with %s remaining requests is used", token["user"].login, token["rest"]
|
"User %s with %s remaining requests is used", token.user.login, token.rest
|
||||||
)
|
)
|
||||||
|
|
||||||
return token["value"]
|
return token.value
|
||||||
|
@ -146,7 +146,7 @@ class PRInfo:
|
|||||||
self.body = github_event["pull_request"]["body"]
|
self.body = github_event["pull_request"]["body"]
|
||||||
self.labels = {
|
self.labels = {
|
||||||
label["name"] for label in github_event["pull_request"]["labels"]
|
label["name"] for label in github_event["pull_request"]["labels"]
|
||||||
}
|
} # type: Set[str]
|
||||||
|
|
||||||
self.user_login = github_event["pull_request"]["user"]["login"]
|
self.user_login = github_event["pull_request"]["user"]["login"]
|
||||||
self.user_orgs = set([])
|
self.user_orgs = set([])
|
||||||
@ -178,7 +178,7 @@ class PRInfo:
|
|||||||
if pull_request is None or pull_request["state"] == "closed":
|
if pull_request is None or pull_request["state"] == "closed":
|
||||||
# it's merged PR to master
|
# it's merged PR to master
|
||||||
self.number = 0
|
self.number = 0
|
||||||
self.labels = {}
|
self.labels = set()
|
||||||
self.pr_html_url = f"{repo_prefix}/commits/{ref}"
|
self.pr_html_url = f"{repo_prefix}/commits/{ref}"
|
||||||
self.base_ref = ref
|
self.base_ref = ref
|
||||||
self.base_name = self.repo_full_name
|
self.base_name = self.repo_full_name
|
||||||
@ -228,7 +228,7 @@ class PRInfo:
|
|||||||
print(json.dumps(github_event, sort_keys=True, indent=4))
|
print(json.dumps(github_event, sort_keys=True, indent=4))
|
||||||
self.sha = os.getenv("GITHUB_SHA")
|
self.sha = os.getenv("GITHUB_SHA")
|
||||||
self.number = 0
|
self.number = 0
|
||||||
self.labels = {}
|
self.labels = set()
|
||||||
repo_prefix = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}"
|
repo_prefix = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}"
|
||||||
self.task_url = GITHUB_RUN_URL
|
self.task_url = GITHUB_RUN_URL
|
||||||
self.commit_html_url = f"{repo_prefix}/commits/{self.sha}"
|
self.commit_html_url = f"{repo_prefix}/commits/{self.sha}"
|
||||||
|
@ -112,7 +112,7 @@ def should_run_checks_for_pr(pr_info: PRInfo) -> Tuple[bool, str, str]:
|
|||||||
return True, "No special conditions apply", "pending"
|
return True, "No special conditions apply", "pending"
|
||||||
|
|
||||||
|
|
||||||
def check_pr_description(pr_info) -> Tuple[str, str]:
|
def check_pr_description(pr_info: PRInfo) -> Tuple[str, str]:
|
||||||
lines = list(
|
lines = list(
|
||||||
map(lambda x: x.strip(), pr_info.body.split("\n") if pr_info.body else [])
|
map(lambda x: x.strip(), pr_info.body.split("\n") if pr_info.body else [])
|
||||||
)
|
)
|
||||||
|
@ -313,7 +313,7 @@ def check_suspicious_changed_files(changed_files):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def approve_run(workflow_description: WorkflowDescription, token):
|
def approve_run(workflow_description: WorkflowDescription, token: str) -> None:
|
||||||
url = f"{workflow_description.api_url}/approve"
|
url = f"{workflow_description.api_url}/approve"
|
||||||
_exec_post_with_retry(url, token)
|
_exec_post_with_retry(url, token)
|
||||||
|
|
||||||
@ -391,7 +391,7 @@ def rerun_workflow(workflow_description, token):
|
|||||||
|
|
||||||
|
|
||||||
def check_workflow_completed(
|
def check_workflow_completed(
|
||||||
event_data, workflow_description: WorkflowDescription, token: str
|
event_data: dict, workflow_description: WorkflowDescription, token: str
|
||||||
) -> bool:
|
) -> bool:
|
||||||
if workflow_description.action == "completed":
|
if workflow_description.action == "completed":
|
||||||
attempt = 0
|
attempt = 0
|
||||||
|
Loading…
Reference in New Issue
Block a user