ClickHouse/tests/ci/style_check.py

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

171 lines
5.4 KiB
Python
Raw Normal View History

2021-09-10 11:52:21 +00:00
#!/usr/bin/env python3
import argparse
import csv
2021-09-10 11:52:21 +00:00
import logging
import os
import subprocess
2021-12-01 14:23:51 +00:00
import sys
from pathlib import Path
from typing import List, Tuple
2021-12-01 14:23:51 +00:00
2023-12-18 12:21:13 +00:00
from docker_images_helper import get_docker_image, pull_image
from env_helper import REPO_COPY, TEMP_PATH
from git_helper import GIT_PREFIX, git_runner
from pr_info import PRInfo
from report import ERROR, FAILURE, SUCCESS, JobReport, TestResults, read_test_results
from ssh import SSHKey
from stopwatch import Stopwatch
2021-09-10 14:27:03 +00:00
def process_result(
2023-09-22 11:16:46 +00:00
result_directory: Path,
) -> Tuple[str, str, TestResults, List[Path]]:
test_results = [] # type: TestResults
2021-09-10 11:52:21 +00:00
additional_files = []
2023-09-22 11:16:46 +00:00
# Just upload all files from result_directory.
2022-01-13 19:52:02 +00:00
# If task provides processed results, then it's responsible
2023-09-22 11:16:46 +00:00
# for content of result_directory.
if result_directory.exists():
additional_files = [p for p in result_directory.iterdir() if p.is_file()]
2021-09-10 11:52:21 +00:00
2021-12-12 12:09:44 +00:00
status = []
2023-09-22 11:16:46 +00:00
status_path = result_directory / "check_status.tsv"
if status_path.exists():
logging.info("Found check_status.tsv")
2022-01-13 19:52:02 +00:00
with open(status_path, "r", encoding="utf-8") as status_file:
status = list(csv.reader(status_file, delimiter="\t"))
2021-09-10 11:52:21 +00:00
if len(status) != 1 or len(status[0]) != 2:
2023-09-22 11:16:46 +00:00
logging.info("Files in result folder %s", os.listdir(result_directory))
return ERROR, "Invalid check_status.tsv", test_results, additional_files
2021-09-10 11:52:21 +00:00
state, description = status[0][0], status[0][1]
try:
2023-09-22 11:16:46 +00:00
results_path = result_directory / "test_results.tsv"
test_results = read_test_results(results_path)
2021-09-10 11:52:21 +00:00
if len(test_results) == 0:
raise Exception("Empty results")
return state, description, test_results, additional_files
except Exception:
if state == SUCCESS:
state, description = ERROR, "Failed to read test_results.tsv"
2021-09-10 11:52:21 +00:00
return state, description, test_results, additional_files
def parse_args():
parser = argparse.ArgumentParser("Check and report style issues in the repository")
parser.add_argument("--push", default=True, help=argparse.SUPPRESS)
parser.add_argument(
"--no-push",
action="store_false",
dest="push",
help="do not commit and push automatic fixes",
default=argparse.SUPPRESS,
)
return parser.parse_args()
def checkout_head(pr_info: PRInfo) -> None:
# It works ONLY for PRs, and only over ssh, so either
# ROBOT_CLICKHOUSE_SSH_KEY should be set or ssh-agent should work
assert pr_info.number
if not pr_info.head_name == pr_info.base_name:
# We can't push to forks, sorry folks
return
remote_url = pr_info.event["pull_request"]["base"]["repo"]["ssh_url"]
fetch_cmd = (
f"{GIT_PREFIX} fetch --depth=1 "
f"{remote_url} {pr_info.head_ref}:head-{pr_info.head_ref}"
)
if os.getenv("ROBOT_CLICKHOUSE_SSH_KEY", ""):
with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"):
git_runner(fetch_cmd)
else:
git_runner(fetch_cmd)
git_runner(f"git checkout -f head-{pr_info.head_ref}")
def commit_push_staged(pr_info: PRInfo) -> None:
# It works ONLY for PRs, and only over ssh, so either
# ROBOT_CLICKHOUSE_SSH_KEY should be set or ssh-agent should work
assert pr_info.number
if not pr_info.head_name == pr_info.base_name:
# We can't push to forks, sorry folks
return
git_staged = git_runner("git diff --cached --name-only")
if not git_staged:
return
remote_url = pr_info.event["pull_request"]["base"]["repo"]["ssh_url"]
git_runner(f"{GIT_PREFIX} commit -m 'Automatic style fix'")
push_cmd = (
f"{GIT_PREFIX} push {remote_url} head-{pr_info.head_ref}:{pr_info.head_ref}"
)
if os.getenv("ROBOT_CLICKHOUSE_SSH_KEY", ""):
with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"):
git_runner(push_cmd)
else:
git_runner(push_cmd)
2023-12-18 12:21:13 +00:00
def checkout_last_ref(pr_info: PRInfo) -> None:
# Checkout the merge commit back to avoid special effects
assert pr_info.number
if not pr_info.head_name == pr_info.base_name:
# We can't push to forks, sorry folks
return
git_runner("git checkout -f -")
def main():
2021-09-10 14:27:03 +00:00
logging.basicConfig(level=logging.INFO)
logging.getLogger("git_helper").setLevel(logging.DEBUG)
args = parse_args()
2021-11-19 14:47:04 +00:00
stopwatch = Stopwatch()
repo_path = Path(REPO_COPY)
2023-10-04 12:28:38 +00:00
temp_path = Path(TEMP_PATH)
2023-09-22 11:16:46 +00:00
temp_path.mkdir(parents=True, exist_ok=True)
2021-09-15 12:59:39 +00:00
2021-11-26 14:00:09 +00:00
pr_info = PRInfo()
2021-09-10 11:52:21 +00:00
IMAGE_NAME = "clickhouse/style-test"
image = pull_image(get_docker_image(IMAGE_NAME))
2022-02-03 13:06:21 +00:00
cmd = (
2022-01-13 19:52:02 +00:00
f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE "
f"--volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output "
f"{image}"
2022-02-03 13:06:21 +00:00
)
2023-12-18 12:21:13 +00:00
if args.push:
checkout_head(pr_info)
2022-02-03 13:06:21 +00:00
logging.info("Is going to run the command: %s", cmd)
subprocess.check_call(
cmd,
2022-01-13 19:52:02 +00:00
shell=True,
)
2022-02-03 13:06:21 +00:00
if args.push:
commit_push_staged(pr_info)
2023-12-18 12:21:13 +00:00
checkout_last_ref(pr_info)
2021-09-15 10:50:30 +00:00
state, description, test_results, additional_files = process_result(temp_path)
2021-11-19 14:47:04 +00:00
JobReport(
description=description,
test_results=test_results,
status=state,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=additional_files,
).dump()
2022-03-29 12:41:47 +00:00
if state in [ERROR, FAILURE]:
print(f"Style check failed: [{description}]")
2022-03-29 12:41:47 +00:00
sys.exit(1)
if __name__ == "__main__":
main()