mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-25 17:12:03 +00:00
Merge pull request #30447 from ClickHouse/trying_actions
Fix s3 for github actions
This commit is contained in:
alesapin
GitHub
commit
2e1ca33311
15
.github/workflows/main.yml
vendored
15
.github/workflows/main.yml
vendored
@ -25,10 +25,6 @@ jobs:
|
||||
uses: actions/checkout@v2
|
||||
- name: Images check
|
||||
run: cd $GITHUB_WORKSPACE/tests/ci && python3 docker_images_check.py
|
||||
env:
|
||||
YANDEX_S3_ACCESS_KEY_ID: ${{ secrets.YANDEX_S3_ACCESS_KEY_ID }}
|
||||
YANDEX_S3_ACCESS_SECRET_KEY: ${{ secrets.YANDEX_S3_ACCESS_SECRET_KEY }}
|
||||
DOCKER_ROBOT_PASSWORD: ${{ secrets.DOCKER_ROBOT_PASSWORD }}
|
||||
- name: Upload images files to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
@ -46,7 +42,12 @@ jobs:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Style Check
|
||||
env:
|
||||
YANDEX_S3_ACCESS_KEY_ID: ${{ secrets.YANDEX_S3_ACCESS_KEY_ID }}
|
||||
YANDEX_S3_ACCESS_SECRET_KEY: ${{ secrets.YANDEX_S3_ACCESS_SECRET_KEY }}
|
||||
run: cd $GITHUB_WORKSPACE/tests/ci && python3 style_check.py
|
||||
FinishCheck:
|
||||
needs: [StyleCheck, DockerHubPush, CheckLabels]
|
||||
runs-on: [self-hosted]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Finish label
|
||||
run: cd $GITHUB_WORKSPACE/tests/ci && python3 finish_check.py
|
||||
|
||||
@ -8,6 +8,7 @@ import os
|
||||
from pr_info import PRInfo
|
||||
from github import Github
|
||||
import shutil
|
||||
from get_robot_token import get_best_robot_token, get_parameter_from_ssm
|
||||
|
||||
NAME = "Push to Dockerhub (actions)"
|
||||
|
||||
@ -167,11 +168,16 @@ def upload_results(s3_client, pr_number, commit_sha, test_results):
|
||||
logging.info("Search result in url %s", url)
|
||||
return url
|
||||
|
||||
def get_commit(gh, commit_sha):
|
||||
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
|
||||
commit = repo.get_commit(commit_sha)
|
||||
return commit
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
repo_path = os.getenv("GITHUB_WORKSPACE", os.path.abspath("../../"))
|
||||
temp_path = os.path.join(os.getenv("RUNNER_TEMP", os.path.abspath("./temp")), 'docker_images_check')
|
||||
dockerhub_password = os.getenv('DOCKER_ROBOT_PASSWORD')
|
||||
dockerhub_password = get_parameter_from_ssm('dockerhub_robot_password')
|
||||
|
||||
if os.path.exists(temp_path):
|
||||
shutil.rmtree(temp_path)
|
||||
@ -207,16 +213,17 @@ if __name__ == "__main__":
|
||||
if len(description) >= 140:
|
||||
description = description[:136] + "..."
|
||||
|
||||
aws_secret_key_id = os.getenv("YANDEX_S3_ACCESS_KEY_ID", "")
|
||||
aws_secret_key = os.getenv("YANDEX_S3_ACCESS_SECRET_KEY", "")
|
||||
|
||||
s3_helper = S3Helper('https://storage.yandexcloud.net', aws_access_key_id=aws_secret_key_id, aws_secret_access_key=aws_secret_key)
|
||||
s3_helper = S3Helper('https://s3.amazonaws.com')
|
||||
|
||||
s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + NAME.lower().replace(' ', '_')
|
||||
status, test_results = process_test_results(s3_helper, images_processing_result, s3_path_prefix)
|
||||
|
||||
url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results)
|
||||
|
||||
gh = Github(get_best_robot_token())
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
commit.create_status(context=NAME, description=description, state=status, target_url=url)
|
||||
|
||||
with open(os.path.join(temp_path, 'changed_images.json'), 'w') as images_file:
|
||||
json.dump(result_images, images_file)
|
||||
|
||||
|
||||
@ -1,12 +1,18 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import boto3
|
||||
from github import Github
|
||||
|
||||
def get_best_robot_token(token_prefix_env_name="ROBOT_TOKEN_", total_tokens=4):
|
||||
def get_parameter_from_ssm(name, decrypt=True, client=None):
|
||||
if not client:
|
||||
client = boto3.client('ssm', region_name='us-east-1')
|
||||
return client.get_parameter(Name=name, WithDecryption=decrypt)['Parameter']['Value']
|
||||
|
||||
def get_best_robot_token(token_prefix_env_name="github_robot_token_", total_tokens=4):
|
||||
client = boto3.client('ssm', region_name='us-east-1')
|
||||
tokens = {}
|
||||
for i in range(total_tokens):
|
||||
for i in range(1, total_tokens + 1):
|
||||
token_name = token_prefix_env_name + str(i)
|
||||
token = os.getenv(token_name)
|
||||
token = get_parameter_from_ssm(token_name, True, client)
|
||||
gh = Github(token)
|
||||
rest, _ = gh.rate_limiting
|
||||
tokens[token] = rest
|
||||
|
||||
@ -9,6 +9,7 @@ from s3_helper import S3Helper
|
||||
from pr_info import PRInfo
|
||||
import shutil
|
||||
import sys
|
||||
from get_robot_token import get_best_robot_token
|
||||
|
||||
NAME = 'PVS Studio (actions)'
|
||||
LICENCE_NAME = 'Free license: ClickHouse, Yandex'
|
||||
@ -39,6 +40,11 @@ def _process_txt_report(path):
|
||||
errors.append(':'.join(line.split('\t')[0:2]))
|
||||
return warnings, errors
|
||||
|
||||
def get_commit(gh, commit_sha):
|
||||
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
|
||||
commit = repo.get_commit(commit_sha)
|
||||
return commit
|
||||
|
||||
def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files):
|
||||
s3_path_prefix = str(pr_number) + "/" + commit_sha + "/" + NAME.lower().replace(' ', '_')
|
||||
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
|
||||
@ -75,8 +81,7 @@ if __name__ == "__main__":
|
||||
# this check modify repository so copy it to the temp directory
|
||||
logging.info("Repo copy path %s", repo_path)
|
||||
|
||||
aws_secret_key_id = os.getenv("YANDEX_S3_ACCESS_KEY_ID", "")
|
||||
aws_secret_key = os.getenv("YANDEX_S3_ACCESS_SECRET_KEY", "")
|
||||
gh = Github(get_best_robot_token())
|
||||
|
||||
images_path = os.path.join(temp_path, 'changed_images.json')
|
||||
docker_image = 'clickhouse/pvs-test'
|
||||
@ -90,10 +95,7 @@ if __name__ == "__main__":
|
||||
|
||||
logging.info("Got docker image %s", docker_image)
|
||||
|
||||
if not aws_secret_key_id or not aws_secret_key:
|
||||
logging.info("No secrets, will not upload anything to S3")
|
||||
|
||||
s3_helper = S3Helper('https://storage.yandexcloud.net', aws_access_key_id=aws_secret_key_id, aws_secret_access_key=aws_secret_key)
|
||||
s3_helper = S3Helper('https://s3.amazonaws.com')
|
||||
|
||||
licence_key = os.getenv('PVS_STUDIO_KEY')
|
||||
cmd = f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --volume={repo_path}:/repo_folder --volume={temp_path}:/test_output -e LICENCE_NAME='{LICENCE_NAME}' -e LICENCE_KEY='{licence_key}' {docker_image}"
|
||||
@ -130,6 +132,8 @@ if __name__ == "__main__":
|
||||
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_logs)
|
||||
|
||||
print("::notice ::Report url: {}".format(report_url))
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
commit.create_status(context=NAME, description=description, state=status, target_url=report_url)
|
||||
except Exception as ex:
|
||||
print("Got an exception", ex)
|
||||
sys.exit(1)
|
||||
|
||||
@ -5,6 +5,8 @@ import requests
|
||||
from pr_info import PRInfo
|
||||
import sys
|
||||
import logging
|
||||
from github import Github
|
||||
from get_robot_token import get_best_robot_token
|
||||
|
||||
NAME = 'Run Check (actions)'
|
||||
|
||||
@ -112,8 +114,13 @@ if __name__ == "__main__":
|
||||
|
||||
pr_info = PRInfo(event, need_orgs=True)
|
||||
can_run, description = should_run_checks_for_pr(pr_info)
|
||||
gh = Github(get_best_robot_token())
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
|
||||
if not can_run:
|
||||
print("::notice ::Cannot run")
|
||||
commit.create_status(context=NAME, description=description, state="failure", target_url=url)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("::notice ::Can run")
|
||||
commit.create_status(context=NAME, description=description, state="pending", target_url=url)
|
||||
|
||||
@ -6,6 +6,7 @@ import boto3
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
from multiprocessing.dummy import Pool
|
||||
from compress_files import compress_file_fast
|
||||
from get_robot_token import get_parameter_from_ssm
|
||||
|
||||
def _md5(fname):
|
||||
hash_md5 = hashlib.md5()
|
||||
@ -27,8 +28,8 @@ def _flatten_list(lst):
|
||||
|
||||
|
||||
class S3Helper(object):
|
||||
def __init__(self, host, aws_access_key_id, aws_secret_access_key):
|
||||
self.session = boto3.session.Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
|
||||
def __init__(self, host):
|
||||
self.session = boto3.session.Session(region_name='us-east-1')
|
||||
self.client = self.session.client('s3', endpoint_url=host)
|
||||
|
||||
def _upload_file_to_s3(self, bucket_name, file_path, s3_path):
|
||||
@ -55,7 +56,7 @@ class S3Helper(object):
|
||||
|
||||
self.client.upload_file(file_path, bucket_name, s3_path, ExtraArgs=metadata)
|
||||
logging.info("Upload {} to {}. Meta: {}".format(file_path, s3_path, metadata))
|
||||
return "https://storage.yandexcloud.net/{bucket}/{path}".format(bucket=bucket_name, path=s3_path)
|
||||
return "https://s3.amazonaws.com/{bucket}/{path}".format(bucket=bucket_name, path=s3_path)
|
||||
|
||||
def upload_test_report_to_s3(self, file_path, s3_path):
|
||||
return self._upload_file_to_s3('clickhouse-test-reports', file_path, s3_path)
|
||||
|
||||
@ -10,6 +10,7 @@ from s3_helper import S3Helper
|
||||
import time
|
||||
import json
|
||||
from pr_info import PRInfo
|
||||
from get_robot_token import get_best_robot_token
|
||||
|
||||
NAME = "Style Check (actions)"
|
||||
|
||||
@ -78,6 +79,12 @@ def upload_results(s3_client, pr_number, commit_sha, test_results, additional_fi
|
||||
logging.info("Search result in url %s", url)
|
||||
return url
|
||||
|
||||
|
||||
def get_commit(gh, commit_sha):
|
||||
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
|
||||
commit = repo.get_commit(commit_sha)
|
||||
return commit
|
||||
|
||||
def update_check_with_curl(check_id):
|
||||
cmd_template = ("curl -v --request PATCH --url https://api.github.com/repos/ClickHouse/ClickHouse/check-runs/{} "
|
||||
"--header 'authorization: Bearer {}' "
|
||||
@ -99,8 +106,7 @@ if __name__ == "__main__":
|
||||
if not os.path.exists(temp_path):
|
||||
os.makedirs(temp_path)
|
||||
|
||||
aws_secret_key_id = os.getenv("YANDEX_S3_ACCESS_KEY_ID", "")
|
||||
aws_secret_key = os.getenv("YANDEX_S3_ACCESS_SECRET_KEY", "")
|
||||
gh = Github(get_best_robot_token())
|
||||
|
||||
images_path = os.path.join(temp_path, 'changed_images.json')
|
||||
docker_image = 'clickhouse/style-test'
|
||||
@ -123,12 +129,11 @@ if __name__ == "__main__":
|
||||
else:
|
||||
raise Exception(f"Cannot pull dockerhub for image {docker_image}")
|
||||
|
||||
if not aws_secret_key_id or not aws_secret_key:
|
||||
logging.info("No secrets, will not upload anything to S3")
|
||||
|
||||
s3_helper = S3Helper('https://storage.yandexcloud.net', aws_access_key_id=aws_secret_key_id, aws_secret_access_key=aws_secret_key)
|
||||
s3_helper = S3Helper('https://s3.amazonaws.com')
|
||||
|
||||
subprocess.check_output(f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE --volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output {docker_image}", shell=True)
|
||||
state, description, test_results, additional_files = process_result(temp_path)
|
||||
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_files)
|
||||
print("::notice ::Report url: {}".format(report_url))
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
commit.create_status(context=NAME, description=description, state=state, target_url=report_url)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user