mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Merge pull request #28851 from ClickHouse/trying_actions
Trying github actions
This commit is contained in:
alesapin
GitHub
commit
9e8dfdb089
66
.github/workflows/main.yml
vendored
Normal file
66
.github/workflows/main.yml
vendored
Normal file
@ -0,0 +1,66 @@
|
||||
name: Ligthweight GithubActions
|
||||
on: # yamllint disable-line rule:truthy
|
||||
pull_request:
|
||||
types:
|
||||
- labeled
|
||||
- unlabeled
|
||||
- synchronize
|
||||
- reopened
|
||||
- opened
|
||||
branches:
|
||||
- master
|
||||
jobs:
|
||||
CheckLabels:
|
||||
runs-on: [self-hosted]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Labels check
|
||||
run: cd $GITHUB_WORKSPACE/tests/ci && python3 run_check.py
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
DockerHubPush:
|
||||
needs: CheckLabels
|
||||
runs-on: [self-hosted]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Images check
|
||||
run: cd $GITHUB_WORKSPACE/tests/ci && python3 docker_images_check.py
|
||||
env:
|
||||
YANDEX_S3_ACCESS_KEY_ID: ${{ secrets.YANDEX_S3_ACCESS_KEY_ID }}
|
||||
YANDEX_S3_ACCESS_SECRET_KEY: ${{ secrets.YANDEX_S3_ACCESS_SECRET_KEY }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
DOCKER_ROBOT_PASSWORD: ${{ secrets.DOCKER_ROBOT_PASSWORD }}
|
||||
- name: Upload images files to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/docker_images_check/changed_images.json
|
||||
StyleCheck:
|
||||
needs: DockerHubPush
|
||||
runs-on: [self-hosted]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/style_check
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Style Check
|
||||
env:
|
||||
YANDEX_S3_ACCESS_KEY_ID: ${{ secrets.YANDEX_S3_ACCESS_KEY_ID }}
|
||||
YANDEX_S3_ACCESS_SECRET_KEY: ${{ secrets.YANDEX_S3_ACCESS_SECRET_KEY }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: cd $GITHUB_WORKSPACE/tests/ci && python3 style_check.py
|
||||
FinishCheck:
|
||||
needs: [StyleCheck, DockerHubPush, CheckLabels]
|
||||
runs-on: [self-hosted]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Finish label
|
||||
run: cd $GITHUB_WORKSPACE/tests/ci && python3 finish_check.py
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@ -37,7 +37,9 @@ RUN set -x \
|
||||
|| echo "WARNING: Some file was just downloaded from the internet without any validation and we are installing it into the system"; } \
|
||||
&& dpkg -i "${PKG_VERSION}.deb"
|
||||
|
||||
CMD echo "Running PVS version $PKG_VERSION" && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \
|
||||
ENV CCACHE_DIR=/test_output/ccache
|
||||
|
||||
CMD echo "Running PVS version $PKG_VERSION" && mkdir -p $CCACHE_DIR && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \
|
||||
&& cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \
|
||||
&& ninja re2_st clickhouse_grpc_protos \
|
||||
&& pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
# yaml check is not the best one
|
||||
|
||||
cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_output/check_status.tsv
|
||||
./check-style -n |& tee /test_output/style_output.txt
|
||||
./check-typos |& tee /test_output/typos_output.txt
|
||||
|
||||
51
tests/ci/compress_files.py
Normal file
51
tests/ci/compress_files.py
Normal file
@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
import subprocess
|
||||
import logging
|
||||
import os
|
||||
|
||||
def compress_file_fast(path, archive_path):
|
||||
if os.path.exists('/usr/bin/pigz'):
|
||||
subprocess.check_call("pigz < {} > {}".format(path, archive_path), shell=True)
|
||||
else:
|
||||
subprocess.check_call("gzip < {} > {}".format(path, archive_path), shell=True)
|
||||
|
||||
|
||||
def compress_fast(path, archive_path, exclude=None):
|
||||
pigz_part = ''
|
||||
if os.path.exists('/usr/bin/pigz'):
|
||||
logging.info("pigz found, will compress and decompress faster")
|
||||
pigz_part = "--use-compress-program='pigz'"
|
||||
else:
|
||||
pigz_part = '-z'
|
||||
logging.info("no pigz, compressing with default tar")
|
||||
|
||||
if exclude is None:
|
||||
exclude_part = ""
|
||||
elif isinstance(exclude, list):
|
||||
exclude_part = " ".join(["--exclude {}".format(x) for x in exclude])
|
||||
else:
|
||||
exclude_part = "--exclude {}".format(str(exclude))
|
||||
|
||||
fname = os.path.basename(path)
|
||||
if os.path.isfile(path):
|
||||
path = os.path.dirname(path)
|
||||
else:
|
||||
path += "/.."
|
||||
cmd = "tar {} {} -cf {} -C {} {}".format(pigz_part, exclude_part, archive_path, path, fname)
|
||||
logging.debug("compress_fast cmd:{}".format(cmd))
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
|
||||
|
||||
def decompress_fast(archive_path, result_path=None):
|
||||
pigz_part = ''
|
||||
if os.path.exists('/usr/bin/pigz'):
|
||||
logging.info("pigz found, will compress and decompress faster ('{}' -> '{}')".format(archive_path, result_path))
|
||||
pigz_part = "--use-compress-program='pigz'"
|
||||
else:
|
||||
pigz_part = '-z'
|
||||
logging.info("no pigz, decompressing with default tar ('{}' -> '{}')".format(archive_path, result_path))
|
||||
|
||||
if result_path is None:
|
||||
subprocess.check_call("tar {} -xf {}".format(pigz_part, archive_path), shell=True)
|
||||
else:
|
||||
subprocess.check_call("tar {} -xf {} -C {}".format(pigz_part, archive_path, result_path), shell=True)
|
||||
233
tests/ci/docker_images_check.py
Normal file
233
tests/ci/docker_images_check.py
Normal file
@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
import subprocess
|
||||
import logging
|
||||
from report import create_test_html_report
|
||||
from s3_helper import S3Helper
|
||||
import json
|
||||
import os
|
||||
from pr_info import PRInfo
|
||||
from github import Github
|
||||
import shutil
|
||||
|
||||
NAME = "Push to Dockerhub (actions)"
|
||||
|
||||
def get_changed_docker_images(pr_info, repo_path, image_file_path):
|
||||
images_dict = {}
|
||||
path_to_images_file = os.path.join(repo_path, image_file_path)
|
||||
if os.path.exists(path_to_images_file):
|
||||
with open(path_to_images_file, 'r') as dict_file:
|
||||
images_dict = json.load(dict_file)
|
||||
else:
|
||||
logging.info("Image file %s doesnt exists in repo %s", image_file_path, repo_path)
|
||||
|
||||
dockerhub_repo_name = 'yandex'
|
||||
if not images_dict:
|
||||
return [], dockerhub_repo_name
|
||||
|
||||
files_changed = pr_info.changed_files
|
||||
|
||||
logging.info("Changed files for PR %s @ %s: %s", pr_info.number, pr_info.sha, str(files_changed))
|
||||
|
||||
changed_images = []
|
||||
|
||||
for dockerfile_dir, image_description in images_dict.items():
|
||||
if image_description['name'].startswith('clickhouse/'):
|
||||
dockerhub_repo_name = 'clickhouse'
|
||||
|
||||
for f in files_changed:
|
||||
if f.startswith(dockerfile_dir):
|
||||
logging.info(
|
||||
"Found changed file '%s' which affects docker image '%s' with path '%s'",
|
||||
f, image_description['name'], dockerfile_dir)
|
||||
changed_images.append(dockerfile_dir)
|
||||
break
|
||||
|
||||
# The order is important: dependents should go later than bases, so that
|
||||
# they are built with updated base versions.
|
||||
index = 0
|
||||
while index < len(changed_images):
|
||||
image = changed_images[index]
|
||||
for dependent in images_dict[image]['dependent']:
|
||||
logging.info(
|
||||
"Marking docker image '%s' as changed because it depends on changed docker image '%s'",
|
||||
dependent, image)
|
||||
changed_images.append(dependent)
|
||||
index += 1
|
||||
if index > 100:
|
||||
# Sanity check to prevent infinite loop.
|
||||
raise "Too many changed docker images, this is a bug." + str(changed_images)
|
||||
|
||||
# If a dependent image was already in the list because its own files
|
||||
# changed, but then it was added as a dependent of a changed base, we
|
||||
# must remove the earlier entry so that it doesn't go earlier than its
|
||||
# base. This way, the dependent will be rebuilt later than the base, and
|
||||
# will correctly use the updated version of the base.
|
||||
seen = set()
|
||||
no_dups_reversed = []
|
||||
for x in reversed(changed_images):
|
||||
if x not in seen:
|
||||
seen.add(x)
|
||||
no_dups_reversed.append(x)
|
||||
|
||||
result = [(x, images_dict[x]['name']) for x in reversed(no_dups_reversed)]
|
||||
logging.info("Changed docker images for PR %s @ %s: '%s'", pr_info.number, pr_info.sha, result)
|
||||
return result, dockerhub_repo_name
|
||||
|
||||
def build_and_push_one_image(path_to_dockerfile_folder, image_name, version_string):
|
||||
logging.info("Building docker image %s with version %s from path %s", image_name, version_string, path_to_dockerfile_folder)
|
||||
build_log = None
|
||||
push_log = None
|
||||
with open('build_log_' + str(image_name).replace('/', '_') + "_" + version_string, 'w') as pl:
|
||||
cmd = "docker build --network=host -t {im}:{ver} {path}".format(im=image_name, ver=version_string, path=path_to_dockerfile_folder)
|
||||
retcode = subprocess.Popen(cmd, shell=True, stderr=pl, stdout=pl).wait()
|
||||
build_log = str(pl.name)
|
||||
if retcode != 0:
|
||||
return False, build_log, None
|
||||
|
||||
with open('tag_log_' + str(image_name).replace('/', '_') + "_" + version_string, 'w') as pl:
|
||||
cmd = "docker build --network=host -t {im} {path}".format(im=image_name, path=path_to_dockerfile_folder)
|
||||
retcode = subprocess.Popen(cmd, shell=True, stderr=pl, stdout=pl).wait()
|
||||
build_log = str(pl.name)
|
||||
if retcode != 0:
|
||||
return False, build_log, None
|
||||
|
||||
logging.info("Pushing image %s to dockerhub", image_name)
|
||||
|
||||
with open('push_log_' + str(image_name).replace('/', '_') + "_" + version_string, 'w') as pl:
|
||||
cmd = "docker push {im}:{ver}".format(im=image_name, ver=version_string)
|
||||
retcode = subprocess.Popen(cmd, shell=True, stderr=pl, stdout=pl).wait()
|
||||
push_log = str(pl.name)
|
||||
if retcode != 0:
|
||||
return False, build_log, push_log
|
||||
|
||||
logging.info("Processing of %s successfully finished", image_name)
|
||||
return True, build_log, push_log
|
||||
|
||||
def process_single_image(versions, path_to_dockerfile_folder, image_name):
|
||||
logging.info("Image will be pushed with versions %s", ', '.join(versions))
|
||||
result = []
|
||||
for ver in versions:
|
||||
for i in range(5):
|
||||
success, build_log, push_log = build_and_push_one_image(path_to_dockerfile_folder, image_name, ver)
|
||||
if success:
|
||||
result.append((image_name + ":" + ver, build_log, push_log, 'OK'))
|
||||
break
|
||||
logging.info("Got error will retry %s time and sleep for %s seconds", i, i * 5)
|
||||
time.sleep(i * 5)
|
||||
else:
|
||||
result.append((image_name + ":" + ver, build_log, push_log, 'FAIL'))
|
||||
|
||||
logging.info("Processing finished")
|
||||
return result
|
||||
|
||||
|
||||
def process_test_results(s3_client, test_results, s3_path_prefix):
|
||||
overall_status = 'success'
|
||||
processed_test_results = []
|
||||
for image, build_log, push_log, status in test_results:
|
||||
if status != 'OK':
|
||||
overall_status = 'failure'
|
||||
url_part = ''
|
||||
if build_log is not None and os.path.exists(build_log):
|
||||
build_url = s3_client.upload_test_report_to_s3(
|
||||
build_log,
|
||||
s3_path_prefix + "/" + os.path.basename(build_log))
|
||||
url_part += '<a href="{}">build_log</a>'.format(build_url)
|
||||
if push_log is not None and os.path.exists(push_log):
|
||||
push_url = s3_client.upload_test_report_to_s3(
|
||||
push_log,
|
||||
s3_path_prefix + "/" + os.path.basename(push_log))
|
||||
if url_part:
|
||||
url_part += ', '
|
||||
url_part += '<a href="{}">push_log</a>'.format(push_url)
|
||||
if url_part:
|
||||
test_name = image + ' (' + url_part + ')'
|
||||
else:
|
||||
test_name = image
|
||||
processed_test_results.append((test_name, status))
|
||||
return overall_status, processed_test_results
|
||||
|
||||
def upload_results(s3_client, pr_number, commit_sha, test_results):
|
||||
s3_path_prefix = f"{pr_number}/{commit_sha}/" + NAME.lower().replace(' ', '_')
|
||||
|
||||
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
|
||||
branch_name = "master"
|
||||
if pr_number != 0:
|
||||
branch_name = "PR #{}".format(pr_number)
|
||||
branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_number)
|
||||
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
|
||||
|
||||
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
|
||||
|
||||
html_report = create_test_html_report(NAME, test_results, "https://hub.docker.com/u/clickhouse", task_url, branch_url, branch_name, commit_url)
|
||||
with open('report.html', 'w') as f:
|
||||
f.write(html_report)
|
||||
|
||||
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
|
||||
logging.info("Search result in url %s", url)
|
||||
return url
|
||||
|
||||
def get_commit(gh, commit_sha):
|
||||
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
|
||||
commit = repo.get_commit(commit_sha)
|
||||
return commit
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
repo_path = os.getenv("GITHUB_WORKSPACE", os.path.abspath("../../"))
|
||||
temp_path = os.path.join(os.getenv("RUNNER_TEMP", os.path.abspath("./temp")), 'docker_images_check')
|
||||
dockerhub_password = os.getenv('DOCKER_ROBOT_PASSWORD')
|
||||
|
||||
if os.path.exists(temp_path):
|
||||
shutil.rmtree(temp_path)
|
||||
|
||||
if not os.path.exists(temp_path):
|
||||
os.makedirs(temp_path)
|
||||
|
||||
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
|
||||
event = json.load(event_file)
|
||||
|
||||
pr_info = PRInfo(event, False, True)
|
||||
changed_images, dockerhub_repo_name = get_changed_docker_images(pr_info, repo_path, "docker/images.json")
|
||||
logging.info("Has changed images %s", ', '.join([str(image[0]) for image in changed_images]))
|
||||
pr_commit_version = str(pr_info.number) + '-' + pr_info.sha
|
||||
|
||||
versions = [str(pr_info.number), pr_commit_version]
|
||||
|
||||
subprocess.check_output("docker login --username 'robotclickhouse' --password '{}'".format(dockerhub_password), shell=True)
|
||||
|
||||
result_images = {}
|
||||
images_processing_result = []
|
||||
for rel_path, image_name in changed_images:
|
||||
full_path = os.path.join(repo_path, rel_path)
|
||||
images_processing_result += process_single_image(versions, full_path, image_name)
|
||||
result_images[image_name] = pr_commit_version
|
||||
|
||||
if len(changed_images):
|
||||
description = "Updated " + ','.join([im[1] for im in changed_images])
|
||||
else:
|
||||
description = "Nothing to update"
|
||||
|
||||
|
||||
if len(description) >= 140:
|
||||
description = description[:136] + "..."
|
||||
|
||||
aws_secret_key_id = os.getenv("YANDEX_S3_ACCESS_KEY_ID", "")
|
||||
aws_secret_key = os.getenv("YANDEX_S3_ACCESS_SECRET_KEY", "")
|
||||
|
||||
s3_helper = S3Helper('https://storage.yandexcloud.net', aws_access_key_id=aws_secret_key_id, aws_secret_access_key=aws_secret_key)
|
||||
|
||||
s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + NAME.lower().replace(' ', '_')
|
||||
status, test_results = process_test_results(s3_helper, images_processing_result, s3_path_prefix)
|
||||
|
||||
url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results)
|
||||
|
||||
gh = Github(os.getenv("GITHUB_TOKEN"))
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
commit.create_status(context=NAME, description=description, state=status, target_url=url)
|
||||
|
||||
with open(os.path.join(temp_path, 'changed_images.json'), 'w') as images_file:
|
||||
json.dump(result_images, images_file)
|
||||
|
||||
print("::notice ::Report url: {}".format(url))
|
||||
print("::set-output name=url_output::\"{}\"".format(url))
|
||||
43
tests/ci/finish_check.py
Normal file
43
tests/ci/finish_check.py
Normal file
@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
from github import Github
|
||||
from pr_info import PRInfo
|
||||
import json
|
||||
import os
|
||||
|
||||
NAME = 'Run Check (actions)'
|
||||
|
||||
def filter_statuses(statuses):
|
||||
"""
|
||||
Squash statuses to latest state
|
||||
1. context="first", state="success", update_time=1
|
||||
2. context="second", state="success", update_time=2
|
||||
3. context="first", stat="failure", update_time=3
|
||||
=========>
|
||||
1. context="second", state="success"
|
||||
2. context="first", stat="failure"
|
||||
"""
|
||||
filt = {}
|
||||
for status in sorted(statuses, key=lambda x: x.updated_at):
|
||||
filt[status.context] = status
|
||||
return filt
|
||||
|
||||
|
||||
def get_commit(gh, commit_sha):
|
||||
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
|
||||
commit = repo.get_commit(commit_sha)
|
||||
return commit
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
|
||||
event = json.load(event_file)
|
||||
|
||||
pr_info = PRInfo(event, need_orgs=True)
|
||||
gh = Github(os.getenv("GITHUB_TOKEN"))
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
|
||||
url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
|
||||
statuses = filter_statuses(list(commit.get_statuses()))
|
||||
if NAME in statuses and statuses[NAME].state == "pending":
|
||||
commit.create_status(context=NAME, description="All checks finished", state="success", target_url=url)
|
||||
13
tests/ci/metrics_lambda/Dockerfile
Normal file
13
tests/ci/metrics_lambda/Dockerfile
Normal file
@ -0,0 +1,13 @@
|
||||
FROM public.ecr.aws/lambda/python:3.9
|
||||
|
||||
# Copy function code
|
||||
COPY app.py ${LAMBDA_TASK_ROOT}
|
||||
|
||||
# Install the function's dependencies using file requirements.txt
|
||||
# from your project folder.
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
|
||||
|
||||
# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
|
||||
CMD [ "app.handler" ]
|
||||
143
tests/ci/metrics_lambda/app.py
Normal file
143
tests/ci/metrics_lambda/app.py
Normal file
@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import requests
|
||||
import argparse
|
||||
import jwt
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
from collections import namedtuple
|
||||
|
||||
def get_key_and_app_from_aws():
|
||||
import boto3
|
||||
secret_name = "clickhouse_github_secret_key"
|
||||
session = boto3.session.Session()
|
||||
client = session.client(
|
||||
service_name='secretsmanager',
|
||||
)
|
||||
get_secret_value_response = client.get_secret_value(
|
||||
SecretId=secret_name
|
||||
)
|
||||
data = json.loads(get_secret_value_response['SecretString'])
|
||||
return data['clickhouse-app-key'], int(data['clickhouse-app-id'])
|
||||
|
||||
def handler(event, context):
|
||||
private_key, app_id = get_key_and_app_from_aws()
|
||||
main(private_key, app_id, True)
|
||||
|
||||
def get_installation_id(jwt_token):
|
||||
headers = {
|
||||
"Authorization": f"Bearer {jwt_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.get("https://api.github.com/app/installations", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data[0]['id']
|
||||
|
||||
def get_access_token(jwt_token, installation_id):
|
||||
headers = {
|
||||
"Authorization": f"Bearer {jwt_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.post(f"https://api.github.com/app/installations/{installation_id}/access_tokens", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data['token']
|
||||
|
||||
|
||||
RunnerDescription = namedtuple('RunnerDescription', ['id', 'name', 'tags', 'offline', 'busy'])
|
||||
|
||||
def list_runners(access_token):
|
||||
headers = {
|
||||
"Authorization": f"token {access_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
|
||||
response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
print("Total runners", data['total_count'])
|
||||
runners = data['runners']
|
||||
result = []
|
||||
for runner in runners:
|
||||
tags = [tag['name'] for tag in runner['labels']]
|
||||
desc = RunnerDescription(id=runner['id'], name=runner['name'], tags=tags,
|
||||
offline=runner['status']=='offline', busy=runner['busy'])
|
||||
result.append(desc)
|
||||
return result
|
||||
|
||||
def push_metrics_to_cloudwatch(listed_runners, namespace):
|
||||
import boto3
|
||||
client = boto3.client('cloudwatch')
|
||||
metrics_data = []
|
||||
busy_runners = sum(1 for runner in listed_runners if runner.busy)
|
||||
metrics_data.append({
|
||||
'MetricName': 'BusyRunners',
|
||||
'Value': busy_runners,
|
||||
'Unit': 'Count',
|
||||
})
|
||||
total_active_runners = sum(1 for runner in listed_runners if not runner.offline)
|
||||
metrics_data.append({
|
||||
'MetricName': 'ActiveRunners',
|
||||
'Value': total_active_runners,
|
||||
'Unit': 'Count',
|
||||
})
|
||||
total_runners = len(listed_runners)
|
||||
metrics_data.append({
|
||||
'MetricName': 'TotalRunners',
|
||||
'Value': total_runners,
|
||||
'Unit': 'Count',
|
||||
})
|
||||
if total_active_runners == 0:
|
||||
busy_ratio = 100
|
||||
else:
|
||||
busy_ratio = busy_runners / total_active_runners * 100
|
||||
|
||||
metrics_data.append({
|
||||
'MetricName': 'BusyRunnersRatio',
|
||||
'Value': busy_ratio,
|
||||
'Unit': 'Percent',
|
||||
})
|
||||
|
||||
client.put_metric_data(Namespace='RunnersMetrics', MetricData=metrics_data)
|
||||
|
||||
def main(github_secret_key, github_app_id, push_to_cloudwatch):
|
||||
payload = {
|
||||
"iat": int(time.time()) - 60,
|
||||
"exp": int(time.time()) + (10 * 60),
|
||||
"iss": github_app_id,
|
||||
}
|
||||
|
||||
encoded_jwt = jwt.encode(payload, github_secret_key, algorithm="RS256")
|
||||
installation_id = get_installation_id(encoded_jwt)
|
||||
access_token = get_access_token(encoded_jwt, installation_id)
|
||||
runners = list_runners(access_token)
|
||||
if push_to_cloudwatch:
|
||||
push_metrics_to_cloudwatch(runners, 'RunnersMetrics')
|
||||
else:
|
||||
print(runners)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='Get list of runners and their states')
|
||||
parser.add_argument('-p', '--private-key-path', help='Path to file with private key')
|
||||
parser.add_argument('-k', '--private-key', help='Private key')
|
||||
parser.add_argument('-a', '--app-id', type=int, help='GitHub application ID', required=True)
|
||||
parser.add_argument('--push-to-cloudwatch', action='store_true', help='Store received token in parameter store')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.private_key_path and not args.private_key:
|
||||
print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
|
||||
|
||||
if args.private_key_path and args.private_key:
|
||||
print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
|
||||
|
||||
if args.private_key:
|
||||
private_key = args.private_key
|
||||
else:
|
||||
with open(args.private_key_path, 'r') as key_file:
|
||||
private_key = key_file.read()
|
||||
|
||||
main(private_key, args.app_id, args.push_to_cloudwatch)
|
||||
3
tests/ci/metrics_lambda/requirements.txt
Normal file
3
tests/ci/metrics_lambda/requirements.txt
Normal file
@ -0,0 +1,3 @@
|
||||
requests
|
||||
PyJWT
|
||||
cryptography
|
||||
41
tests/ci/pr_info.py
Normal file
41
tests/ci/pr_info.py
Normal file
@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
import requests
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import urllib
|
||||
from unidiff import PatchSet
|
||||
|
||||
|
||||
class PRInfo:
|
||||
def __init__(self, github_event, need_orgs=False, need_changed_files=False):
|
||||
self.number = github_event['number']
|
||||
if 'after' in github_event:
|
||||
self.sha = github_event['after']
|
||||
else:
|
||||
self.sha = github_event['pull_request']['head']['sha']
|
||||
|
||||
self.labels = set([l['name'] for l in github_event['pull_request']['labels']])
|
||||
self.user_login = github_event['pull_request']['user']['login']
|
||||
self.user_orgs = set([])
|
||||
if need_orgs:
|
||||
user_orgs_response = requests.get(github_event['pull_request']['user']['organizations_url'])
|
||||
if user_orgs_response.ok:
|
||||
response_json = user_orgs_response.json()
|
||||
self.user_orgs = set(org['id'] for org in response_json)
|
||||
|
||||
self.changed_files = set([])
|
||||
if need_changed_files:
|
||||
diff_url = github_event['pull_request']['diff_url']
|
||||
diff = urllib.request.urlopen(github_event['pull_request']['diff_url'])
|
||||
diff_object = PatchSet(diff, diff.headers.get_charsets()[0])
|
||||
self.changed_files = set([f.path for f in diff_object])
|
||||
|
||||
def get_dict(self):
|
||||
return {
|
||||
'sha': self.sha,
|
||||
'number': self.number,
|
||||
'labels': self.labels,
|
||||
'user_login': self.user_login,
|
||||
'user_orgs': self.user_orgs,
|
||||
}
|
||||
144
tests/ci/pvs_check.py
Normal file
144
tests/ci/pvs_check.py
Normal file
@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
import subprocess
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
from github import Github
|
||||
from report import create_test_html_report
|
||||
from s3_helper import S3Helper
|
||||
from pr_info import PRInfo
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
NAME = 'PVS Studio (actions)'
|
||||
LICENCE_NAME = 'Free license: ClickHouse, Yandex'
|
||||
HTML_REPORT_FOLDER = 'pvs-studio-html-report'
|
||||
TXT_REPORT_NAME = 'pvs-studio-task-report.txt'
|
||||
|
||||
def process_logs(s3_client, additional_logs, s3_path_prefix):
|
||||
additional_urls = []
|
||||
for log_path in additional_logs:
|
||||
if log_path:
|
||||
additional_urls.append(
|
||||
s3_client.upload_test_report_to_s3(
|
||||
log_path,
|
||||
s3_path_prefix + "/" + os.path.basename(log_path)))
|
||||
|
||||
return additional_urls
|
||||
|
||||
def _process_txt_report(path):
|
||||
warnings = []
|
||||
errors = []
|
||||
with open(path, 'r') as report_file:
|
||||
for line in report_file:
|
||||
if 'viva64' in line:
|
||||
continue
|
||||
elif 'warn' in line:
|
||||
warnings.append(':'.join(line.split('\t')[0:2]))
|
||||
elif 'err' in line:
|
||||
errors.append(':'.join(line.split('\t')[0:2]))
|
||||
return warnings, errors
|
||||
|
||||
def get_commit(gh, commit_sha):
|
||||
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
|
||||
commit = repo.get_commit(commit_sha)
|
||||
return commit
|
||||
|
||||
def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files):
|
||||
s3_path_prefix = str(pr_number) + "/" + commit_sha + "/" + NAME.lower().replace(' ', '_')
|
||||
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
|
||||
|
||||
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
|
||||
branch_name = "master"
|
||||
if pr_number != 0:
|
||||
branch_name = "PR #{}".format(pr_number)
|
||||
branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_number)
|
||||
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
|
||||
|
||||
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
|
||||
|
||||
raw_log_url = additional_urls[0]
|
||||
additional_urls.pop(0)
|
||||
|
||||
html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls)
|
||||
with open('report.html', 'w') as f:
|
||||
f.write(html_report)
|
||||
|
||||
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
|
||||
logging.info("Search result in url %s", url)
|
||||
return url
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
repo_path = os.path.join(os.getenv("REPO_COPY", os.path.abspath("../../")))
|
||||
temp_path = os.path.join(os.getenv("RUNNER_TEMP", os.path.abspath("./temp")), 'pvs_check')
|
||||
|
||||
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
|
||||
event = json.load(event_file)
|
||||
pr_info = PRInfo(event)
|
||||
# this check modify repository so copy it to the temp directory
|
||||
logging.info("Repo copy path %s", repo_path)
|
||||
|
||||
aws_secret_key_id = os.getenv("YANDEX_S3_ACCESS_KEY_ID", "")
|
||||
aws_secret_key = os.getenv("YANDEX_S3_ACCESS_SECRET_KEY", "")
|
||||
|
||||
gh = Github(os.getenv("GITHUB_TOKEN"))
|
||||
|
||||
images_path = os.path.join(temp_path, 'changed_images.json')
|
||||
docker_image = 'clickhouse/pvs-test'
|
||||
if os.path.exists(images_path):
|
||||
logging.info("Images file exists")
|
||||
with open(images_path, 'r') as images_fd:
|
||||
images = json.load(images_fd)
|
||||
logging.info("Got images %s", images)
|
||||
if 'clickhouse/pvs-test' in images:
|
||||
docker_image += ':' + images['clickhouse/pvs-test']
|
||||
|
||||
logging.info("Got docker image %s", docker_image)
|
||||
|
||||
if not aws_secret_key_id or not aws_secret_key:
|
||||
logging.info("No secrets, will not upload anything to S3")
|
||||
|
||||
s3_helper = S3Helper('https://storage.yandexcloud.net', aws_access_key_id=aws_secret_key_id, aws_secret_access_key=aws_secret_key)
|
||||
|
||||
licence_key = os.getenv('PVS_STUDIO_KEY')
|
||||
cmd = f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --volume={repo_path}:/repo_folder --volume={temp_path}:/test_output -e LICENCE_NAME='{LICENCE_NAME}' -e LICENCE_KEY='{licence_key}' {docker_image}"
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
|
||||
try:
|
||||
subprocess.check_output(cmd, shell=True)
|
||||
except:
|
||||
commit.create_status(context=NAME, description='PVS report failed to build', state='failure', target_url=f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + NAME.lower().replace(' ', '_')
|
||||
html_urls = s3_helper.upload_test_folder_to_s3(os.path.join(temp_path, HTML_REPORT_FOLDER), s3_path_prefix)
|
||||
index_html = None
|
||||
|
||||
for url in html_urls:
|
||||
if 'index.html' in url:
|
||||
index_html = '<a href="{}">HTML report</a>'.format(url)
|
||||
break
|
||||
|
||||
if not index_html:
|
||||
commit.create_status(context=NAME, description='PVS report failed to build', state='failure', target_url=f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}")
|
||||
sys.exit(1)
|
||||
|
||||
txt_report = os.path.join(temp_path, TXT_REPORT_NAME)
|
||||
warnings, errors = _process_txt_report(txt_report)
|
||||
errors = errors + warnings
|
||||
|
||||
status = 'success'
|
||||
test_results = [(index_html, "Look at the report"), ("Errors count not checked", "OK")]
|
||||
description = "Total errors {}".format(len(errors))
|
||||
additional_logs = [txt_report, os.path.join(temp_path, 'pvs-studio.log')]
|
||||
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_logs)
|
||||
|
||||
print("::notice ::Report url: {}".format(report_url))
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
commit.create_status(context=NAME, description=description, state=status, target_url=report_url)
|
||||
except Exception as ex:
|
||||
print("Got an exception", ex)
|
||||
sys.exit(1)
|
||||
298
tests/ci/report.py
Normal file
298
tests/ci/report.py
Normal file
@ -0,0 +1,298 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import datetime
|
||||
|
||||
### FIXME: BEST FRONTEND PRACTICIES BELOW
|
||||
|
||||
HTML_BASE_TEST_TEMPLATE = """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<style>
|
||||
@font-face {{
|
||||
font-family:'Yandex Sans Display Web';
|
||||
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot);
|
||||
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot?#iefix) format('embedded-opentype'),
|
||||
url(https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2) format('woff2'),
|
||||
url(https://yastatic.net/adv-www/_/v2Sve_obH3rKm6rKrtSQpf-eB7U.woff) format('woff'),
|
||||
url(https://yastatic.net/adv-www/_/PzD8hWLMunow5i3RfJ6WQJAL7aI.ttf) format('truetype'),
|
||||
url(https://yastatic.net/adv-www/_/lF_KG5g4tpQNlYIgA0e77fBSZ5s.svg#YandexSansDisplayWeb-Regular) format('svg');
|
||||
font-weight:400;
|
||||
font-style:normal;
|
||||
font-stretch:normal
|
||||
}}
|
||||
|
||||
body {{ font-family: "Yandex Sans Display Web", Arial, sans-serif; background: #EEE; }}
|
||||
h1 {{ margin-left: 10px; }}
|
||||
th, td {{ border: 0; padding: 5px 10px 5px 10px; text-align: left; vertical-align: top; line-height: 1.5; background-color: #FFF;
|
||||
td {{ white-space: pre; font-family: Monospace, Courier New; }}
|
||||
border: 0; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }}
|
||||
a {{ color: #06F; text-decoration: none; }}
|
||||
a:hover, a:active {{ color: #F40; text-decoration: underline; }}
|
||||
table {{ border: 0; }}
|
||||
.main {{ margin-left: 10%; }}
|
||||
p.links a {{ padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-space: nowrap; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }}
|
||||
th {{ cursor: pointer; }}
|
||||
|
||||
</style>
|
||||
<title>{title}</title>
|
||||
</head>
|
||||
<body>
|
||||
<div class="main">
|
||||
|
||||
<h1>{header}</h1>
|
||||
<p class="links">
|
||||
<a href="{raw_log_url}">{raw_log_name}</a>
|
||||
<a href="{commit_url}">Commit</a>
|
||||
{additional_urls}
|
||||
<a href="{task_url}">Task (github actions)</a>
|
||||
</p>
|
||||
{test_part}
|
||||
</body>
|
||||
<script type="text/javascript">
|
||||
/// Straight from https://stackoverflow.com/questions/14267781/sorting-html-table-with-javascript
|
||||
|
||||
const getCellValue = (tr, idx) => tr.children[idx].innerText || tr.children[idx].textContent;
|
||||
|
||||
const comparer = (idx, asc) => (a, b) => ((v1, v2) =>
|
||||
v1 !== '' && v2 !== '' && !isNaN(v1) && !isNaN(v2) ? v1 - v2 : v1.toString().localeCompare(v2)
|
||||
)(getCellValue(asc ? a : b, idx), getCellValue(asc ? b : a, idx));
|
||||
|
||||
// do the work...
|
||||
document.querySelectorAll('th').forEach(th => th.addEventListener('click', (() => {{
|
||||
const table = th.closest('table');
|
||||
Array.from(table.querySelectorAll('tr:nth-child(n+2)'))
|
||||
.sort(comparer(Array.from(th.parentNode.children).indexOf(th), this.asc = !this.asc))
|
||||
.forEach(tr => table.appendChild(tr) );
|
||||
}})));
|
||||
</script>
|
||||
</html>
|
||||
"""
|
||||
|
||||
HTML_TEST_PART = """
|
||||
<table>
|
||||
<tr>
|
||||
{headers}
|
||||
</tr>
|
||||
{rows}
|
||||
</table>
|
||||
"""
|
||||
|
||||
BASE_HEADERS = ['Test name', 'Test status']
|
||||
|
||||
|
||||
def _format_header(header, branch_name, branch_url=None):
|
||||
result = ' '.join([w.capitalize() for w in header.split(' ')])
|
||||
result = result.replace("Clickhouse", "ClickHouse")
|
||||
result = result.replace("clickhouse", "ClickHouse")
|
||||
if 'ClickHouse' not in result:
|
||||
result = 'ClickHouse ' + result
|
||||
result += ' for '
|
||||
if branch_url:
|
||||
result += '<a href="{url}">{name}</a>'.format(url=branch_url, name=branch_name)
|
||||
else:
|
||||
result += branch_name
|
||||
return result
|
||||
|
||||
|
||||
def _get_status_style(status):
|
||||
style = "font-weight: bold;"
|
||||
if status in ('OK', 'success', 'PASSED'):
|
||||
style += 'color: #0A0;'
|
||||
elif status in ('FAIL', 'failure', 'error', 'FAILED', 'Timeout'):
|
||||
style += 'color: #F00;'
|
||||
else:
|
||||
style += 'color: #FFB400;'
|
||||
return style
|
||||
|
||||
|
||||
def _get_html_url(url):
|
||||
if isinstance(url, str):
|
||||
return '<a href="{url}">{name}</a>'.format(url=url, name=os.path.basename(url))
|
||||
if isinstance(url, tuple):
|
||||
return '<a href="{url}">{name}</a>'.format(url=url[0], name=url[1])
|
||||
return ''
|
||||
|
||||
|
||||
def create_test_html_report(header, test_result, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls=[]):
|
||||
if test_result:
|
||||
rows_part = ""
|
||||
num_fails = 0
|
||||
has_test_time = False
|
||||
has_test_logs = False
|
||||
for result in test_result:
|
||||
test_name = result[0]
|
||||
test_status = result[1]
|
||||
|
||||
test_logs = None
|
||||
test_time = None
|
||||
if len(result) > 2:
|
||||
test_time = result[2]
|
||||
has_test_time = True
|
||||
|
||||
if len(result) > 3:
|
||||
test_logs = result[3]
|
||||
has_test_logs = True
|
||||
|
||||
row = "<tr>"
|
||||
row += "<td>" + test_name + "</td>"
|
||||
style = _get_status_style(test_status)
|
||||
|
||||
# Allow to quickly scroll to the first failure.
|
||||
is_fail = test_status == "FAIL" or test_status == 'FLAKY'
|
||||
is_fail_id = ""
|
||||
if is_fail:
|
||||
num_fails = num_fails + 1
|
||||
is_fail_id = 'id="fail' + str(num_fails) + '" '
|
||||
|
||||
row += '<td ' + is_fail_id + 'style="{}">'.format(style) + test_status + "</td>"
|
||||
|
||||
if test_time is not None:
|
||||
row += "<td>" + test_time + "</td>"
|
||||
|
||||
if test_logs is not None:
|
||||
test_logs_html = "<br>".join([_get_html_url(url) for url in test_logs])
|
||||
row += "<td>" + test_logs_html + "</td>"
|
||||
|
||||
row += "</tr>"
|
||||
rows_part += row
|
||||
|
||||
headers = BASE_HEADERS
|
||||
if has_test_time:
|
||||
headers.append('Test time, sec.')
|
||||
if has_test_logs:
|
||||
headers.append('Logs')
|
||||
|
||||
headers = ''.join(['<th>' + h + '</th>' for h in headers])
|
||||
test_part = HTML_TEST_PART.format(headers=headers, rows=rows_part)
|
||||
else:
|
||||
test_part = ""
|
||||
|
||||
additional_html_urls = ""
|
||||
for url in additional_urls:
|
||||
additional_html_urls += ' ' + _get_html_url(url)
|
||||
|
||||
result = HTML_BASE_TEST_TEMPLATE.format(
|
||||
title=_format_header(header, branch_name),
|
||||
header=_format_header(header, branch_name, branch_url),
|
||||
raw_log_name=os.path.basename(raw_log_url),
|
||||
raw_log_url=raw_log_url,
|
||||
task_url=task_url,
|
||||
test_part=test_part,
|
||||
branch_name=branch_name,
|
||||
commit_url=commit_url,
|
||||
additional_urls=additional_html_urls
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
HTML_BASE_BUILD_TEMPLATE = """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
@font-face {{
|
||||
font-family:'Yandex Sans Display Web';
|
||||
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot);
|
||||
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot?#iefix) format('embedded-opentype'),
|
||||
url(https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2) format('woff2'),
|
||||
url(https://yastatic.net/adv-www/_/v2Sve_obH3rKm6rKrtSQpf-eB7U.woff) format('woff'),
|
||||
url(https://yastatic.net/adv-www/_/PzD8hWLMunow5i3RfJ6WQJAL7aI.ttf) format('truetype'),
|
||||
url(https://yastatic.net/adv-www/_/lF_KG5g4tpQNlYIgA0e77fBSZ5s.svg#YandexSansDisplayWeb-Regular) format('svg');
|
||||
font-weight:400;
|
||||
font-style:normal;
|
||||
font-stretch:normal
|
||||
}}
|
||||
|
||||
body {{ font-family: "Yandex Sans Display Web", Arial, sans-serif; background: #EEE; }}
|
||||
h1 {{ margin-left: 10px; }}
|
||||
th, td {{ border: 0; padding: 5px 10px 5px 10px; text-align: left; vertical-align: top; line-height: 1.5; background-color: #FFF;
|
||||
border: 0; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }}
|
||||
a {{ color: #06F; text-decoration: none; }}
|
||||
a:hover, a:active {{ color: #F40; text-decoration: underline; }}
|
||||
table {{ border: 0; }}
|
||||
.main {{ margin: auto; }}
|
||||
p.links a {{ padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-space: nowrap; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }}
|
||||
tr:hover td {{filter: brightness(95%);}}
|
||||
</style>
|
||||
<title>{title}</title>
|
||||
</head>
|
||||
<body>
|
||||
<div class="main">
|
||||
<h1>{header}</h1>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Compiler</th>
|
||||
<th>Build type</th>
|
||||
<th>Sanitizer</th>
|
||||
<th>Bundled</th>
|
||||
<th>Splitted</th>
|
||||
<th>Status</th>
|
||||
<th>Build log</th>
|
||||
<th>Build time</th>
|
||||
<th class="artifacts">Artifacts</th>
|
||||
</tr>
|
||||
{rows}
|
||||
</table>
|
||||
<p class="links">
|
||||
<a href="{commit_url}">Commit</a>
|
||||
<a href="{task_url}">Task (private network)</a>
|
||||
</p>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
LINK_TEMPLATE = '<a href="{url}">{text}</a>'
|
||||
|
||||
|
||||
def create_build_html_report(header, build_results, build_logs_urls, artifact_urls_list, task_url, branch_url, branch_name, commit_url):
|
||||
rows = ""
|
||||
for (build_result, build_log_url, artifact_urls) in zip(build_results, build_logs_urls, artifact_urls_list):
|
||||
row = "<tr>"
|
||||
row += "<td>{}</td>".format(build_result.compiler)
|
||||
if build_result.build_type:
|
||||
row += "<td>{}</td>".format(build_result.build_type)
|
||||
else:
|
||||
row += "<td>{}</td>".format("relwithdebuginfo")
|
||||
if build_result.sanitizer:
|
||||
row += "<td>{}</td>".format(build_result.sanitizer)
|
||||
else:
|
||||
row += "<td>{}</td>".format("none")
|
||||
|
||||
row += "<td>{}</td>".format(build_result.bundled)
|
||||
row += "<td>{}</td>".format(build_result.splitted)
|
||||
|
||||
if build_result.status:
|
||||
style = _get_status_style(build_result.status)
|
||||
row += '<td style="{}">{}</td>'.format(style, build_result.status)
|
||||
else:
|
||||
style = _get_status_style("error")
|
||||
row += '<td style="{}">{}</td>'.format(style, "error")
|
||||
|
||||
row += '<td><a href="{}">link</a></td>'.format(build_log_url)
|
||||
|
||||
if build_result.elapsed_seconds:
|
||||
delta = datetime.timedelta(seconds=build_result.elapsed_seconds)
|
||||
else:
|
||||
delta = 'unknown'
|
||||
|
||||
row += '<td>{}</td>'.format(str(delta))
|
||||
|
||||
links = ""
|
||||
link_separator = "<br/>"
|
||||
if artifact_urls:
|
||||
for artifact_url in artifact_urls:
|
||||
links += LINK_TEMPLATE.format(text=os.path.basename(artifact_url), url=artifact_url)
|
||||
links += link_separator
|
||||
if links:
|
||||
links = links[:-len(link_separator)]
|
||||
row += "<td>{}</td>".format(links)
|
||||
|
||||
row += "</tr>"
|
||||
rows += row
|
||||
return HTML_BASE_BUILD_TEMPLATE.format(
|
||||
title=_format_header(header, branch_name),
|
||||
header=_format_header(header, branch_name, branch_url),
|
||||
rows=rows,
|
||||
task_url=task_url,
|
||||
branch_name=branch_name,
|
||||
commit_url=commit_url)
|
||||
125
tests/ci/run_check.py
Normal file
125
tests/ci/run_check.py
Normal file
@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import json
|
||||
import requests
|
||||
from pr_info import PRInfo
|
||||
import sys
|
||||
import logging
|
||||
from github import Github
|
||||
|
||||
NAME = 'Run Check (actions)'
|
||||
|
||||
TRUSTED_ORG_IDS = {
|
||||
7409213, # yandex
|
||||
28471076, # altinity
|
||||
54801242, # clickhouse
|
||||
}
|
||||
|
||||
OK_TEST_LABEL = set(["can be tested", "release", "pr-documentation", "pr-doc-fix"])
|
||||
DO_NOT_TEST_LABEL = "do not test"
|
||||
|
||||
# Individual trusted contirbutors who are not in any trusted organization.
|
||||
# Can be changed in runtime: we will append users that we learned to be in
|
||||
# a trusted org, to save GitHub API calls.
|
||||
TRUSTED_CONTRIBUTORS = {
|
||||
"achimbab",
|
||||
"adevyatova ", # DOCSUP
|
||||
"Algunenano", # Raúl Marín, Tinybird
|
||||
"AnaUvarova", # DOCSUP
|
||||
"anauvarova", # technical writer, Yandex
|
||||
"annvsh", # technical writer, Yandex
|
||||
"atereh", # DOCSUP
|
||||
"azat",
|
||||
"bharatnc", # Newbie, but already with many contributions.
|
||||
"bobrik", # Seasoned contributor, CloundFlare
|
||||
"BohuTANG",
|
||||
"damozhaeva", # DOCSUP
|
||||
"den-crane",
|
||||
"gyuton", # DOCSUP
|
||||
"gyuton", # technical writer, Yandex
|
||||
"hagen1778", # Roman Khavronenko, seasoned contributor
|
||||
"hczhcz",
|
||||
"hexiaoting", # Seasoned contributor
|
||||
"ildus", # adjust, ex-pgpro
|
||||
"javisantana", # a Spanish ClickHouse enthusiast, ex-Carto
|
||||
"ka1bi4", # DOCSUP
|
||||
"kirillikoff", # DOCSUP
|
||||
"kitaisreal", # Seasoned contributor
|
||||
"kreuzerkrieg",
|
||||
"lehasm", # DOCSUP
|
||||
"michon470", # DOCSUP
|
||||
"MyroTk", # Tester in Altinity
|
||||
"myrrc", # Michael Kot, Altinity
|
||||
"nikvas0",
|
||||
"nvartolomei",
|
||||
"olgarev", # DOCSUP
|
||||
"otrazhenia", # Yandex docs contractor
|
||||
"pdv-ru", # DOCSUP
|
||||
"podshumok", # cmake expert from QRator Labs
|
||||
"s-mx", # Maxim Sabyanin, former employee, present contributor
|
||||
"sevirov", # technical writer, Yandex
|
||||
"spongedu", # Seasoned contributor
|
||||
"ucasFL", # Amos Bird's friend
|
||||
"vdimir", # Employee
|
||||
"vzakaznikov",
|
||||
"YiuRULE",
|
||||
"zlobober" # Developer of YT
|
||||
}
|
||||
|
||||
|
||||
def pr_is_by_trusted_user(pr_user_login, pr_user_orgs):
|
||||
if pr_user_login in TRUSTED_CONTRIBUTORS:
|
||||
logging.info("User '{}' is trusted".format(pr_user_login))
|
||||
return True
|
||||
|
||||
logging.info("User '{}' is not trusted".format(pr_user_login))
|
||||
|
||||
for org_id in pr_user_orgs:
|
||||
if org_id in TRUSTED_ORG_IDS:
|
||||
logging.info("Org '{}' is trusted; will mark user {} as trusted".format(org_id, pr_user_login))
|
||||
return True
|
||||
logging.info("Org '{}' is not trusted".format(org_id))
|
||||
|
||||
return False
|
||||
|
||||
# Returns whether we should look into individual checks for this PR. If not, it
|
||||
# can be skipped entirely.
|
||||
def should_run_checks_for_pr(pr_info):
|
||||
# Consider the labels and whether the user is trusted.
|
||||
force_labels = set(['force tests']).intersection(pr_info.labels)
|
||||
if force_labels:
|
||||
return True, "Labeled '{}'".format(', '.join(force_labels))
|
||||
|
||||
if 'do not test' in pr_info.labels:
|
||||
return False, "Labeled 'do not test'"
|
||||
|
||||
if 'can be tested' not in pr_info.labels and not pr_is_by_trusted_user(pr_info.user_login, pr_info.user_orgs):
|
||||
return False, "Needs 'can be tested' label"
|
||||
|
||||
if 'release' in pr_info.labels or 'pr-backport' in pr_info.labels or 'pr-cherrypick' in pr_info.labels:
|
||||
return False, "Don't try new checks for release/backports/cherry-picks"
|
||||
|
||||
return True, "No special conditions apply"
|
||||
|
||||
def get_commit(gh, commit_sha):
|
||||
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
|
||||
commit = repo.get_commit(commit_sha)
|
||||
return commit
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
|
||||
event = json.load(event_file)
|
||||
|
||||
pr_info = PRInfo(event, need_orgs=True)
|
||||
can_run, description = should_run_checks_for_pr(pr_info)
|
||||
gh = Github(os.getenv("GITHUB_TOKEN"))
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
|
||||
if not can_run:
|
||||
print("::notice ::Cannot run")
|
||||
commit.create_status(context=NAME, description=description, state="failure", target_url=url)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("::notice ::Can run")
|
||||
commit.create_status(context=NAME, description=description, state="pending", target_url=url)
|
||||
99
tests/ci/s3_helper.py
Normal file
99
tests/ci/s3_helper.py
Normal file
@ -0,0 +1,99 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
from multiprocessing.dummy import Pool
|
||||
from compress_files import compress_file_fast
|
||||
|
||||
def _md5(fname):
|
||||
hash_md5 = hashlib.md5()
|
||||
with open(fname, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
hash_md5.update(chunk)
|
||||
logging.debug("MD5 for {} is {}".format(fname, hash_md5.hexdigest()))
|
||||
return hash_md5.hexdigest()
|
||||
|
||||
|
||||
def _flatten_list(lst):
|
||||
result = []
|
||||
for elem in lst:
|
||||
if isinstance(elem, list):
|
||||
result += _flatten_list(elem)
|
||||
else:
|
||||
result.append(elem)
|
||||
return result
|
||||
|
||||
|
||||
class S3Helper(object):
|
||||
def __init__(self, host, aws_access_key_id, aws_secret_access_key):
|
||||
self.session = boto3.session.Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
|
||||
self.client = self.session.client('s3', endpoint_url=host)
|
||||
|
||||
def _upload_file_to_s3(self, bucket_name, file_path, s3_path):
|
||||
logging.debug("Start uploading {} to bucket={} path={}".format(file_path, bucket_name, s3_path))
|
||||
metadata = {}
|
||||
if os.path.getsize(file_path) < 64 * 1024 * 1024:
|
||||
if s3_path.endswith("txt") or s3_path.endswith("log") or s3_path.endswith("err") or s3_path.endswith("out"):
|
||||
metadata['ContentType'] = "text/plain; charset=utf-8"
|
||||
logging.info("Content type %s for file path %s", "text/plain; charset=utf-8", file_path)
|
||||
elif s3_path.endswith("html"):
|
||||
metadata['ContentType'] = "text/html; charset=utf-8"
|
||||
logging.info("Content type %s for file path %s", "text/html; charset=utf-8", file_path)
|
||||
else:
|
||||
logging.info("No content type provied for %s", file_path)
|
||||
else:
|
||||
if s3_path.endswith("txt") or s3_path.endswith("log") or s3_path.endswith("err") or s3_path.endswith("out"):
|
||||
logging.info("Going to compress file log file %s to %s", file_path, file_path + ".gz")
|
||||
compress_file_fast(file_path, file_path + ".gz")
|
||||
file_path += ".gz"
|
||||
s3_path += ".gz"
|
||||
else:
|
||||
logging.info("Processing file without compression")
|
||||
logging.info("File is too large, do not provide content type")
|
||||
|
||||
self.client.upload_file(file_path, bucket_name, s3_path, ExtraArgs=metadata)
|
||||
logging.info("Upload {} to {}. Meta: {}".format(file_path, s3_path, metadata))
|
||||
return "https://storage.yandexcloud.net/{bucket}/{path}".format(bucket=bucket_name, path=s3_path)
|
||||
|
||||
def upload_test_report_to_s3(self, file_path, s3_path):
|
||||
return self._upload_file_to_s3('clickhouse-test-reports', file_path, s3_path)
|
||||
|
||||
def upload_build_file_to_s3(self, file_path, s3_path):
|
||||
return self._upload_file_to_s3('clickhouse-builds', file_path, s3_path)
|
||||
|
||||
def _upload_folder_to_s3(self, folder_path, s3_folder_path, bucket_name, keep_dirs_in_s3_path, upload_symlinks):
|
||||
logging.info("Upload folder '{}' to bucket={} of s3 folder '{}'".format(folder_path, bucket_name, s3_folder_path))
|
||||
if not os.path.exists(folder_path):
|
||||
return []
|
||||
files = os.listdir(folder_path)
|
||||
if not files:
|
||||
return []
|
||||
|
||||
p = Pool(min(len(files), 5))
|
||||
|
||||
def task(file_name):
|
||||
full_fs_path = os.path.join(folder_path, file_name)
|
||||
if keep_dirs_in_s3_path:
|
||||
full_s3_path = s3_folder_path + "/" + os.path.basename(folder_path)
|
||||
else:
|
||||
full_s3_path = s3_folder_path
|
||||
|
||||
if os.path.isdir(full_fs_path):
|
||||
return self._upload_folder_to_s3(full_fs_path, full_s3_path, bucket_name, keep_dirs_in_s3_path, upload_symlinks)
|
||||
|
||||
if os.path.islink(full_fs_path):
|
||||
if upload_symlinks:
|
||||
return self._upload_file_to_s3(bucket_name, full_fs_path, full_s3_path + "/" + file_name)
|
||||
return []
|
||||
|
||||
return self._upload_file_to_s3(bucket_name, full_fs_path, full_s3_path + "/" + file_name)
|
||||
|
||||
return sorted(_flatten_list(list(p.map(task, files))))
|
||||
|
||||
def upload_build_folder_to_s3(self, folder_path, s3_folder_path, keep_dirs_in_s3_path=True, upload_symlinks=True):
|
||||
return self._upload_folder_to_s3(folder_path, s3_folder_path, 'clickhouse-builds', keep_dirs_in_s3_path, upload_symlinks)
|
||||
|
||||
def upload_test_folder_to_s3(self, folder_path, s3_folder_path):
|
||||
return self._upload_folder_to_s3(folder_path, s3_folder_path, 'clickhouse-test-reports', True, True)
|
||||
144
tests/ci/style_check.py
Normal file
144
tests/ci/style_check.py
Normal file
@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
from github import Github
|
||||
from report import create_test_html_report
|
||||
import shutil
|
||||
import logging
|
||||
import subprocess
|
||||
import os
|
||||
import csv
|
||||
from s3_helper import S3Helper
|
||||
import time
|
||||
import json
|
||||
from pr_info import PRInfo
|
||||
|
||||
NAME = "Style Check (actions)"
|
||||
|
||||
|
||||
def process_logs(s3_client, additional_logs, s3_path_prefix):
|
||||
additional_urls = []
|
||||
for log_path in additional_logs:
|
||||
if log_path:
|
||||
additional_urls.append(
|
||||
s3_client.upload_test_report_to_s3(
|
||||
log_path,
|
||||
s3_path_prefix + "/" + os.path.basename(log_path)))
|
||||
|
||||
return additional_urls
|
||||
|
||||
|
||||
def process_result(result_folder):
|
||||
test_results = []
|
||||
additional_files = []
|
||||
# Just upload all files from result_folder.
|
||||
# If task provides processed results, then it's responsible for content of result_folder.
|
||||
if os.path.exists(result_folder):
|
||||
test_files = [f for f in os.listdir(result_folder) if os.path.isfile(os.path.join(result_folder, f))]
|
||||
additional_files = [os.path.join(result_folder, f) for f in test_files]
|
||||
|
||||
status_path = os.path.join(result_folder, "check_status.tsv")
|
||||
logging.info("Found test_results.tsv")
|
||||
status = list(csv.reader(open(status_path, 'r'), delimiter='\t'))
|
||||
if len(status) != 1 or len(status[0]) != 2:
|
||||
return "error", "Invalid check_status.tsv", test_results, additional_files
|
||||
state, description = status[0][0], status[0][1]
|
||||
|
||||
try:
|
||||
results_path = os.path.join(result_folder, "test_results.tsv")
|
||||
test_results = list(csv.reader(open(results_path, 'r'), delimiter='\t'))
|
||||
if len(test_results) == 0:
|
||||
raise Exception("Empty results")
|
||||
|
||||
return state, description, test_results, additional_files
|
||||
except Exception:
|
||||
if state == "success":
|
||||
state, description = "error", "Failed to read test_results.tsv"
|
||||
return state, description, test_results, additional_files
|
||||
|
||||
def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files):
|
||||
s3_path_prefix = f"{pr_number}/{commit_sha}/style_check"
|
||||
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
|
||||
|
||||
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
|
||||
branch_name = "master"
|
||||
if pr_number != 0:
|
||||
branch_name = "PR #{}".format(pr_number)
|
||||
branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_number)
|
||||
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
|
||||
|
||||
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
|
||||
|
||||
raw_log_url = additional_urls[0]
|
||||
additional_urls.pop(0)
|
||||
|
||||
html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls)
|
||||
with open('report.html', 'w') as f:
|
||||
f.write(html_report)
|
||||
|
||||
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
|
||||
logging.info("Search result in url %s", url)
|
||||
return url
|
||||
|
||||
|
||||
def get_commit(gh, commit_sha):
|
||||
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
|
||||
commit = repo.get_commit(commit_sha)
|
||||
return commit
|
||||
|
||||
def update_check_with_curl(check_id):
|
||||
cmd_template = ("curl -v --request PATCH --url https://api.github.com/repos/ClickHouse/ClickHouse/check-runs/{} "
|
||||
"--header 'authorization: Bearer {}' "
|
||||
"--header 'Accept: application/vnd.github.v3+json' "
|
||||
"--header 'content-type: application/json' "
|
||||
"-d '{{\"name\" : \"hello-world-name\"}}'")
|
||||
cmd = cmd_template.format(check_id, os.getenv("GITHUB_TOKEN"))
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
repo_path = os.path.join(os.getenv("GITHUB_WORKSPACE", os.path.abspath("../../")))
|
||||
temp_path = os.path.join(os.getenv("RUNNER_TEMP", os.path.abspath("./temp")), 'style_check')
|
||||
|
||||
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
|
||||
event = json.load(event_file)
|
||||
pr_info = PRInfo(event)
|
||||
|
||||
if not os.path.exists(temp_path):
|
||||
os.makedirs(temp_path)
|
||||
|
||||
aws_secret_key_id = os.getenv("YANDEX_S3_ACCESS_KEY_ID", "")
|
||||
aws_secret_key = os.getenv("YANDEX_S3_ACCESS_SECRET_KEY", "")
|
||||
|
||||
gh = Github(os.getenv("GITHUB_TOKEN"))
|
||||
|
||||
images_path = os.path.join(temp_path, 'changed_images.json')
|
||||
docker_image = 'clickhouse/style-test'
|
||||
if os.path.exists(images_path):
|
||||
logging.info("Images file exists")
|
||||
with open(images_path, 'r') as images_fd:
|
||||
images = json.load(images_fd)
|
||||
logging.info("Got images %s", images)
|
||||
if 'clickhouse/style-test' in images:
|
||||
docker_image += ':' + images['clickhouse/style-test']
|
||||
|
||||
logging.info("Got docker image %s", docker_image)
|
||||
for i in range(10):
|
||||
try:
|
||||
subprocess.check_output(f"docker pull {docker_image}", shell=True)
|
||||
break
|
||||
except Exception as ex:
|
||||
time.sleep(i * 3)
|
||||
logging.info("Got execption pulling docker %s", ex)
|
||||
else:
|
||||
raise Exception(f"Cannot pull dockerhub for image {docker_image}")
|
||||
|
||||
if not aws_secret_key_id or not aws_secret_key:
|
||||
logging.info("No secrets, will not upload anything to S3")
|
||||
|
||||
s3_helper = S3Helper('https://storage.yandexcloud.net', aws_access_key_id=aws_secret_key_id, aws_secret_access_key=aws_secret_key)
|
||||
|
||||
subprocess.check_output(f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE --volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output {docker_image}", shell=True)
|
||||
state, description, test_results, additional_files = process_result(temp_path)
|
||||
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_files)
|
||||
print("::notice ::Report url: {}".format(report_url))
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
commit.create_status(context=NAME, description=description, state=state, target_url=report_url)
|
||||
13
tests/ci/termination_lambda/Dockerfile
Normal file
13
tests/ci/termination_lambda/Dockerfile
Normal file
@ -0,0 +1,13 @@
|
||||
FROM public.ecr.aws/lambda/python:3.9
|
||||
|
||||
# Copy function code
|
||||
COPY app.py ${LAMBDA_TASK_ROOT}
|
||||
|
||||
# Install the function's dependencies using file requirements.txt
|
||||
# from your project folder.
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
|
||||
|
||||
# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
|
||||
CMD [ "app.handler" ]
|
||||
275
tests/ci/termination_lambda/app.py
Normal file
275
tests/ci/termination_lambda/app.py
Normal file
@ -0,0 +1,275 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import requests
|
||||
import argparse
|
||||
import jwt
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
from collections import namedtuple
|
||||
|
||||
def get_key_and_app_from_aws():
|
||||
import boto3
|
||||
secret_name = "clickhouse_github_secret_key"
|
||||
session = boto3.session.Session()
|
||||
client = session.client(
|
||||
service_name='secretsmanager',
|
||||
)
|
||||
get_secret_value_response = client.get_secret_value(
|
||||
SecretId=secret_name
|
||||
)
|
||||
data = json.loads(get_secret_value_response['SecretString'])
|
||||
return data['clickhouse-app-key'], int(data['clickhouse-app-id'])
|
||||
|
||||
def get_installation_id(jwt_token):
|
||||
headers = {
|
||||
"Authorization": f"Bearer {jwt_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.get("https://api.github.com/app/installations", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data[0]['id']
|
||||
|
||||
def get_access_token(jwt_token, installation_id):
|
||||
headers = {
|
||||
"Authorization": f"Bearer {jwt_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.post(f"https://api.github.com/app/installations/{installation_id}/access_tokens", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data['token']
|
||||
|
||||
|
||||
RunnerDescription = namedtuple('RunnerDescription', ['id', 'name', 'tags', 'offline', 'busy'])
|
||||
|
||||
def list_runners(access_token):
|
||||
headers = {
|
||||
"Authorization": f"token {access_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
|
||||
response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
print("Total runners", data['total_count'])
|
||||
runners = data['runners']
|
||||
result = []
|
||||
for runner in runners:
|
||||
tags = [tag['name'] for tag in runner['labels']]
|
||||
desc = RunnerDescription(id=runner['id'], name=runner['name'], tags=tags,
|
||||
offline=runner['status']=='offline', busy=runner['busy'])
|
||||
result.append(desc)
|
||||
return result
|
||||
|
||||
def push_metrics_to_cloudwatch(listed_runners, namespace):
|
||||
import boto3
|
||||
client = boto3.client('cloudwatch')
|
||||
metrics_data = []
|
||||
busy_runners = sum(1 for runner in listed_runners if runner.busy)
|
||||
metrics_data.append({
|
||||
'MetricName': 'BusyRunners',
|
||||
'Value': busy_runners,
|
||||
'Unit': 'Count',
|
||||
})
|
||||
total_active_runners = sum(1 for runner in listed_runners if not runner.offline)
|
||||
metrics_data.append({
|
||||
'MetricName': 'ActiveRunners',
|
||||
'Value': total_active_runners,
|
||||
'Unit': 'Count',
|
||||
})
|
||||
total_runners = len(listed_runners)
|
||||
metrics_data.append({
|
||||
'MetricName': 'TotalRunners',
|
||||
'Value': total_runners,
|
||||
'Unit': 'Count',
|
||||
})
|
||||
if total_active_runners == 0:
|
||||
busy_ratio = 100
|
||||
else:
|
||||
busy_ratio = busy_runners / total_active_runners * 100
|
||||
|
||||
metrics_data.append({
|
||||
'MetricName': 'BusyRunnersRatio',
|
||||
'Value': busy_ratio,
|
||||
'Unit': 'Percent',
|
||||
})
|
||||
|
||||
client.put_metric_data(Namespace='RunnersMetrics', MetricData=metrics_data)
|
||||
|
||||
|
||||
def how_many_instances_to_kill(event_data):
|
||||
data_array = event_data['CapacityToTerminate']
|
||||
to_kill_by_zone = {}
|
||||
for av_zone in data_array:
|
||||
zone_name = av_zone['AvailabilityZone']
|
||||
to_kill = av_zone['Capacity']
|
||||
if zone_name not in to_kill_by_zone:
|
||||
to_kill_by_zone[zone_name] = 0
|
||||
|
||||
to_kill_by_zone[zone_name] += to_kill
|
||||
return to_kill_by_zone
|
||||
|
||||
def get_candidates_to_be_killed(event_data):
|
||||
data_array = event_data['Instances']
|
||||
instances_by_zone = {}
|
||||
for instance in data_array:
|
||||
zone_name = instance['AvailabilityZone']
|
||||
instance_id = instance['InstanceId']
|
||||
if zone_name not in instances_by_zone:
|
||||
instances_by_zone[zone_name] = []
|
||||
instances_by_zone[zone_name].append(instance_id)
|
||||
|
||||
return instances_by_zone
|
||||
|
||||
def delete_runner(access_token, runner):
|
||||
headers = {
|
||||
"Authorization": f"token {access_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
|
||||
response = requests.delete(f"https://api.github.com/orgs/ClickHouse/actions/runners/{runner.id}", headers=headers)
|
||||
response.raise_for_status()
|
||||
print(f"Response code deleting {runner.name} is {response.status_code}")
|
||||
return response.status_code == 204
|
||||
|
||||
|
||||
def main(github_secret_key, github_app_id, event):
|
||||
print("Got event", json.dumps(event, sort_keys=True, indent=4))
|
||||
to_kill_by_zone = how_many_instances_to_kill(event)
|
||||
instances_by_zone = get_candidates_to_be_killed(event)
|
||||
|
||||
payload = {
|
||||
"iat": int(time.time()) - 60,
|
||||
"exp": int(time.time()) + (10 * 60),
|
||||
"iss": github_app_id,
|
||||
}
|
||||
|
||||
encoded_jwt = jwt.encode(payload, github_secret_key, algorithm="RS256")
|
||||
installation_id = get_installation_id(encoded_jwt)
|
||||
access_token = get_access_token(encoded_jwt, installation_id)
|
||||
|
||||
runners = list_runners(access_token)
|
||||
|
||||
to_delete_runners = []
|
||||
instances_to_kill = []
|
||||
for zone in to_kill_by_zone:
|
||||
num_to_kill = to_kill_by_zone[zone]
|
||||
candidates = instances_by_zone[zone]
|
||||
if num_to_kill > len(candidates):
|
||||
raise Exception(f"Required to kill {num_to_kill}, but have only {len(candidates)} candidates in AV {zone}")
|
||||
|
||||
delete_for_av = []
|
||||
for candidate in candidates:
|
||||
if candidate not in set([runner.name for runner in runners]):
|
||||
print(f"Candidate {candidate} was not in runners list, simply delete it")
|
||||
instances_to_kill.append(candidate)
|
||||
|
||||
for candidate in candidates:
|
||||
if len(delete_for_av) + len(instances_to_kill) == num_to_kill:
|
||||
break
|
||||
if candidate in instances_to_kill:
|
||||
continue
|
||||
|
||||
for runner in runners:
|
||||
if runner.name == candidate:
|
||||
if not runner.busy:
|
||||
print(f"Runner {runner.name} is not busy and can be deleted from AV {zone}")
|
||||
delete_for_av.append(runner)
|
||||
else:
|
||||
print(f"Runner {runner.name} is busy, not going to delete it")
|
||||
break
|
||||
|
||||
if len(delete_for_av) < num_to_kill:
|
||||
print(f"Checked all candidates for av {zone}, get to delete {len(delete_for_av)}, but still cannot get required {num_to_kill}")
|
||||
to_delete_runners += delete_for_av
|
||||
|
||||
print("Got instances to kill: ", ', '.join(instances_to_kill))
|
||||
print("Going to delete runners:", ', '.join([runner.name for runner in to_delete_runners]))
|
||||
for runner in to_delete_runners:
|
||||
if delete_runner(access_token, runner):
|
||||
print(f"Runner {runner.name} successfuly deleted from github")
|
||||
instances_to_kill.append(runner.name)
|
||||
else:
|
||||
print(f"Cannot delete {runner.name} from github")
|
||||
|
||||
## push metrics
|
||||
#runners = list_runners(access_token)
|
||||
#push_metrics_to_cloudwatch(runners, 'RunnersMetrics')
|
||||
|
||||
response = {
|
||||
"InstanceIDs": instances_to_kill
|
||||
}
|
||||
print(response)
|
||||
return response
|
||||
|
||||
def handler(event, context):
|
||||
private_key, app_id = get_key_and_app_from_aws()
|
||||
return main(private_key, app_id, event)
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='Get list of runners and their states')
|
||||
parser.add_argument('-p', '--private-key-path', help='Path to file with private key')
|
||||
parser.add_argument('-k', '--private-key', help='Private key')
|
||||
parser.add_argument('-a', '--app-id', type=int, help='GitHub application ID', required=True)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.private_key_path and not args.private_key:
|
||||
print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
|
||||
|
||||
if args.private_key_path and args.private_key:
|
||||
print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
|
||||
|
||||
if args.private_key:
|
||||
private_key = args.private_key
|
||||
else:
|
||||
with open(args.private_key_path, 'r') as key_file:
|
||||
private_key = key_file.read()
|
||||
|
||||
sample_event = {
|
||||
"AutoScalingGroupARN": "arn:aws:autoscaling:us-east-1:<account-id>:autoScalingGroup:d4738357-2d40-4038-ae7e-b00ae0227003:autoScalingGroupName/my-asg",
|
||||
"AutoScalingGroupName": "my-asg",
|
||||
"CapacityToTerminate": [
|
||||
{
|
||||
"AvailabilityZone": "us-east-1b",
|
||||
"Capacity": 1,
|
||||
"InstanceMarketOption": "OnDemand"
|
||||
},
|
||||
{
|
||||
"AvailabilityZone": "us-east-1c",
|
||||
"Capacity": 2,
|
||||
"InstanceMarketOption": "OnDemand"
|
||||
}
|
||||
],
|
||||
"Instances": [
|
||||
{
|
||||
"AvailabilityZone": "us-east-1b",
|
||||
"InstanceId": "i-08d0b3c1a137e02a5",
|
||||
"InstanceType": "t2.nano",
|
||||
"InstanceMarketOption": "OnDemand"
|
||||
},
|
||||
{
|
||||
"AvailabilityZone": "us-east-1c",
|
||||
"InstanceId": "ip-172-31-45-253.eu-west-1.compute.internal",
|
||||
"InstanceType": "t2.nano",
|
||||
"InstanceMarketOption": "OnDemand"
|
||||
},
|
||||
{
|
||||
"AvailabilityZone": "us-east-1c",
|
||||
"InstanceId": "ip-172-31-27-227.eu-west-1.compute.internal",
|
||||
"InstanceType": "t2.nano",
|
||||
"InstanceMarketOption": "OnDemand"
|
||||
},
|
||||
{
|
||||
"AvailabilityZone": "us-east-1c",
|
||||
"InstanceId": "ip-172-31-45-253.eu-west-1.compute.internal",
|
||||
"InstanceType": "t2.nano",
|
||||
"InstanceMarketOption": "OnDemand"
|
||||
}
|
||||
],
|
||||
"Cause": "SCALE_IN"
|
||||
}
|
||||
|
||||
main(private_key, args.app_id, sample_event)
|
||||
3
tests/ci/termination_lambda/requirements.txt
Normal file
3
tests/ci/termination_lambda/requirements.txt
Normal file
@ -0,0 +1,3 @@
|
||||
requests
|
||||
PyJWT
|
||||
cryptography
|
||||
13
tests/ci/token_lambda/Dockerfile
Normal file
13
tests/ci/token_lambda/Dockerfile
Normal file
@ -0,0 +1,13 @@
|
||||
FROM public.ecr.aws/lambda/python:3.9
|
||||
|
||||
# Copy function code
|
||||
COPY app.py ${LAMBDA_TASK_ROOT}
|
||||
|
||||
# Install the function's dependencies using file requirements.txt
|
||||
# from your project folder.
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
|
||||
|
||||
# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
|
||||
CMD [ "app.handler" ]
|
||||
106
tests/ci/token_lambda/app.py
Normal file
106
tests/ci/token_lambda/app.py
Normal file
@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import requests
|
||||
import argparse
|
||||
import jwt
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
|
||||
def get_installation_id(jwt_token):
|
||||
headers = {
|
||||
"Authorization": f"Bearer {jwt_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.get("https://api.github.com/app/installations", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data[0]['id']
|
||||
|
||||
def get_access_token(jwt_token, installation_id):
|
||||
headers = {
|
||||
"Authorization": f"Bearer {jwt_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.post(f"https://api.github.com/app/installations/{installation_id}/access_tokens", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data['token']
|
||||
|
||||
def get_runner_registration_token(access_token):
|
||||
headers = {
|
||||
"Authorization": f"token {access_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.post("https://api.github.com/orgs/ClickHouse/actions/runners/registration-token", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data['token']
|
||||
|
||||
def get_key_and_app_from_aws():
|
||||
import boto3
|
||||
secret_name = "clickhouse_github_secret_key"
|
||||
session = boto3.session.Session()
|
||||
client = session.client(
|
||||
service_name='secretsmanager',
|
||||
)
|
||||
get_secret_value_response = client.get_secret_value(
|
||||
SecretId=secret_name
|
||||
)
|
||||
data = json.loads(get_secret_value_response['SecretString'])
|
||||
return data['clickhouse-app-key'], int(data['clickhouse-app-id'])
|
||||
|
||||
|
||||
def main(github_secret_key, github_app_id, push_to_ssm, ssm_parameter_name):
|
||||
payload = {
|
||||
"iat": int(time.time()) - 60,
|
||||
"exp": int(time.time()) + (10 * 60),
|
||||
"iss": github_app_id,
|
||||
}
|
||||
|
||||
encoded_jwt = jwt.encode(payload, github_secret_key, algorithm="RS256")
|
||||
installation_id = get_installation_id(encoded_jwt)
|
||||
access_token = get_access_token(encoded_jwt, installation_id)
|
||||
runner_registration_token = get_runner_registration_token(access_token)
|
||||
|
||||
if push_to_ssm:
|
||||
import boto3
|
||||
|
||||
print("Trying to put params into ssm manager")
|
||||
client = boto3.client('ssm')
|
||||
client.put_parameter(
|
||||
Name=ssm_parameter_name,
|
||||
Value=runner_registration_token,
|
||||
Type='SecureString',
|
||||
Overwrite=True)
|
||||
else:
|
||||
print("Not push token to AWS Parameter Store, just print:", runner_registration_token)
|
||||
|
||||
|
||||
def handler(event, context):
|
||||
private_key, app_id = get_key_and_app_from_aws()
|
||||
main(private_key, app_id, True, 'github_runner_registration_token')
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='Get new token from github to add runners')
|
||||
parser.add_argument('-p', '--private-key-path', help='Path to file with private key')
|
||||
parser.add_argument('-k', '--private-key', help='Private key')
|
||||
parser.add_argument('-a', '--app-id', type=int, help='GitHub application ID', required=True)
|
||||
parser.add_argument('--push-to-ssm', action='store_true', help='Store received token in parameter store')
|
||||
parser.add_argument('--ssm-parameter-name', default='github_runner_registration_token', help='AWS paramater store parameter name')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.private_key_path and not args.private_key:
|
||||
print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
|
||||
|
||||
if args.private_key_path and args.private_key:
|
||||
print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
|
||||
|
||||
if args.private_key:
|
||||
private_key = args.private_key
|
||||
else:
|
||||
with open(args.private_key_path, 'r') as key_file:
|
||||
private_key = key_file.read()
|
||||
|
||||
main(private_key, args.app_id, args.push_to_ssm, args.ssm_parameter_name)
|
||||
3
tests/ci/token_lambda/requirements.txt
Normal file
3
tests/ci/token_lambda/requirements.txt
Normal file
@ -0,0 +1,3 @@
|
||||
requests
|
||||
PyJWT
|
||||
cryptography
|
||||
20
tests/ci/worker/init.sh
Normal file
20
tests/ci/worker/init.sh
Normal file
@ -0,0 +1,20 @@
|
||||
#!/usr/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
echo "Running init script"
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
export RUNNER_HOME=/home/ubuntu/actions-runner
|
||||
|
||||
echo "Receiving token"
|
||||
export RUNNER_TOKEN=`/usr/local/bin/aws ssm get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value`
|
||||
export RUNNER_URL="https://github.com/ClickHouse"
|
||||
# Funny fact, but metadata service has fixed IP
|
||||
export INSTANCE_ID=`curl -s http://169.254.169.254/latest/meta-data/instance-id`
|
||||
|
||||
cd $RUNNER_HOME
|
||||
|
||||
echo "Going to configure runner"
|
||||
sudo -u ubuntu ./config.sh --url $RUNNER_URL --token $RUNNER_TOKEN --name $INSTANCE_ID --runnergroup Default --labels 'self-hosted,Linux,X64' --work _work
|
||||
|
||||
echo "Run"
|
||||
sudo -u ubuntu ./run.sh
|
||||
47
tests/ci/worker/ubuntu_ami.sh
Normal file
47
tests/ci/worker/ubuntu_ami.sh
Normal file
@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
echo "Running prepare script"
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
export RUNNER_VERSION=2.283.1
|
||||
export RUNNER_HOME=/home/ubuntu/actions-runner
|
||||
|
||||
apt-get update
|
||||
|
||||
apt-get install --yes --no-install-recommends \
|
||||
apt-transport-https \
|
||||
ca-certificates \
|
||||
curl \
|
||||
gnupg \
|
||||
lsb-release \
|
||||
python3-pip \
|
||||
unzip
|
||||
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
|
||||
|
||||
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
|
||||
|
||||
apt-get update
|
||||
|
||||
apt-get install --yes --no-install-recommends docker-ce docker-ce-cli containerd.io
|
||||
|
||||
usermod -aG docker ubuntu
|
||||
|
||||
pip install boto3 pygithub requests urllib3 unidiff
|
||||
|
||||
mkdir -p $RUNNER_HOME && cd $RUNNER_HOME
|
||||
|
||||
curl -O -L https://github.com/actions/runner/releases/download/v$RUNNER_VERSION/actions-runner-linux-x64-$RUNNER_VERSION.tar.gz
|
||||
|
||||
tar xzf ./actions-runner-linux-x64-$RUNNER_VERSION.tar.gz
|
||||
rm -f ./actions-runner-linux-x64-$RUNNER_VERSION.tar.gz
|
||||
./bin/installdependencies.sh
|
||||
|
||||
chown -R ubuntu:ubuntu $RUNNER_HOME
|
||||
|
||||
cd /home/ubuntu
|
||||
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
|
||||
unzip awscliv2.zip
|
||||
./aws/install
|
||||
|
||||
rm -rf /home/ubuntu/awscliv2.zip /home/ubuntu/aws
|
||||
@ -70,7 +70,7 @@ find $ROOT_PATH/{src,base,programs,utils} -name '*.xml' |
|
||||
xargs xmllint --noout --nonet
|
||||
|
||||
# FIXME: for now only clickhouse-test
|
||||
pylint --rcfile=$ROOT_PATH/.pylintrc --score=n $ROOT_PATH/tests/clickhouse-test
|
||||
pylint --rcfile=$ROOT_PATH/.pylintrc --persistent=no --score=n $ROOT_PATH/tests/clickhouse-test
|
||||
|
||||
find $ROOT_PATH -not -path $ROOT_PATH'/contrib*' \( -name '*.yaml' -or -name '*.yml' \) -type f |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
@ -162,7 +162,7 @@ find $ROOT_PATH -name '.gitmodules' | while read i; do grep -F 'url = ' $i | gre
|
||||
find $ROOT_PATH/{src,base,programs} -name '*.h' -or -name '*.cpp' 2>/dev/null | xargs grep -i -F 'General Public License' && echo "There shouldn't be any code snippets under GPL or LGPL"
|
||||
|
||||
# There shouldn't be any docker containers outside docker directory
|
||||
find $ROOT_PATH -not -path $ROOT_PATH'/docker*' -not -path $ROOT_PATH'/contrib*' -name Dockerfile -type f 2>/dev/null | xargs --no-run-if-empty -n1 echo "Please move Dockerfile to docker directory:"
|
||||
find $ROOT_PATH -not -path $ROOT_PATH'/tests/ci*' -not -path $ROOT_PATH'/docker*' -not -path $ROOT_PATH'/contrib*' -name Dockerfile -type f 2>/dev/null | xargs --no-run-if-empty -n1 echo "Please move Dockerfile to docker directory:"
|
||||
|
||||
# There shouldn't be any docker compose files outside docker directory
|
||||
#find $ROOT_PATH -not -path $ROOT_PATH'/tests/testflows*' -not -path $ROOT_PATH'/docker*' -not -path $ROOT_PATH'/contrib*' -name '*compose*.yml' -type f 2>/dev/null | xargs --no-run-if-empty grep -l "version:" | xargs --no-run-if-empty -n1 echo "Please move docker compose to docker directory:"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user