diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b738cf96b46..3d1c9730f99 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,7 +21,6 @@ jobs: python3 run_check.py DockerHubPush: needs: CheckLabels - if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }} runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -58,8 +57,34 @@ jobs: docker kill $(docker ps -q) ||: docker rm -f $(docker ps -a -q) ||: sudo rm -fr $TEMP_PATH + DocsCheck: + needs: DockerHubPush + runs-on: [self-hosted, func-tester] + steps: + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ runner.temp }}/docs_check + - name: Check out repository code + uses: actions/checkout@v2 + - name: Docs Check + env: + TEMP_PATH: ${{runner.temp}}/docs_check + REPO_COPY: ${{runner.temp}}/docs_check/ClickHouse + run: | + cp -r $GITHUB_WORKSPACE $TEMP_PATH + cd $REPO_COPY/tests/ci + python3 docs_check.py + - name: Cleanup + if: always() + run: | + docker kill $(docker ps -q) ||: + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr $TEMP_PATH BuilderDebDebug: needs: DockerHubPush + if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }} runs-on: [self-hosted, builder] steps: - name: Download changed images @@ -184,6 +209,7 @@ jobs: sudo rm -fr $TEMP_PATH FastTest: needs: DockerHubPush + if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }} runs-on: [self-hosted, builder] steps: - name: Check out repository code diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..a396e188327 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,50 @@ +name: ReleaseChecks +concurrency: + group: master-release + cancel-in-progress: true +on: # yamllint disable-line rule:truthy + push: + branches: + - master +jobs: + DockerHubPush: + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Images check + run: | + cd $GITHUB_WORKSPACE/tests/ci + python3 docker_images_check.py + - name: Upload images files to artifacts + uses: actions/upload-artifact@v2 + with: + name: changed_images + path: ${{ runner.temp }}/docker_images_check/changed_images.json + DocsRelease: + needs: DockerHubPush + runs: [self-hosted, func-tester] + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{runner.temp}}/docs_release + - name: Docs Release + env: + TEMP_PATH: ${{runner.temp}}/docs_release + REPO_COPY: ${{runner.temp}}/docs_release/ClickHouse + CLOUDFLARE_TOKEN: ${{secrets.CLOUDFLARE}} + ROBOT_CLICKHOUSE_SSH_KEY: ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}} + run: | + cp -r $GITHUB_WORKSPACE $TEMP_PATH + cd $REPO_COPY/tests/ci + python3 docs_release.py + - name: Cleanup + if: always() + run: | + docker kill $(docker ps -q) ||: + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr $TEMP_PATH diff --git a/docker/docs/builder/Dockerfile b/docker/docs/builder/Dockerfile new file mode 100644 index 00000000000..8afddefa41a --- /dev/null +++ b/docker/docs/builder/Dockerfile @@ -0,0 +1,43 @@ +# docker build -t clickhouse/docs-build . +FROM ubuntu:20.04 + +ENV LANG=C.UTF-8 + +RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \ + python3-setuptools \ + virtualenv \ + wget \ + bash \ + python \ + curl \ + python3-requests \ + sudo \ + git \ + openssl \ + python3-pip \ + software-properties-common \ + language-pack-zh* \ + chinese* \ + fonts-arphic-ukai \ + fonts-arphic-uming \ + fonts-ipafont-mincho \ + fonts-ipafont-gothic \ + fonts-unfonts-core \ + xvfb \ + nodejs \ + npm \ + openjdk-11-jdk \ + ssh-client \ + && pip --no-cache-dir install scipy \ + && apt-get autoremove --yes \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN wget 'https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb' + +RUN npm i -g purify-css + +RUN pip3 install --ignore-installed --upgrade setuptools pip virtualenv diff --git a/docker/docs/check/Dockerfile b/docker/docs/check/Dockerfile new file mode 100644 index 00000000000..55647df5c3e --- /dev/null +++ b/docker/docs/check/Dockerfile @@ -0,0 +1,9 @@ +# docker build -t clickhouse/docs-check . +FROM clickhouse/docs-builder + +COPY run.sh / + +ENV REPO_PATH=/repo_path +ENV OUTPUT_PATH=/output_path + +CMD ["/bin/bash", "/run.sh"] diff --git a/docker/docs/check/run.sh b/docker/docs/check/run.sh new file mode 100644 index 00000000000..f70f82aeb4c --- /dev/null +++ b/docker/docs/check/run.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd $REPO_PATH/docs/tools +mkdir venv +virtualenv -p $(which python3) venv +source venv/bin/activate +python3 -m pip install --ignore-installed -r requirements.txt +./build.py --skip-git-log 2>&1 | tee $OUTPUT_PATH/output.log diff --git a/docker/docs/release/Dockerfile b/docker/docs/release/Dockerfile new file mode 100644 index 00000000000..63765180a4c --- /dev/null +++ b/docker/docs/release/Dockerfile @@ -0,0 +1,9 @@ +# docker build -t clickhouse/docs-release . +FROM clickhouse/docs-builder + +COPY run.sh / + +ENV REPO_PATH=/repo_path +ENV OUTPUT_PATH=/output_path + +CMD ["/bin/bash", "/run.sh"] diff --git a/docker/docs/release/run.sh b/docker/docs/release/run.sh new file mode 100644 index 00000000000..3ecfd26cb44 --- /dev/null +++ b/docker/docs/release/run.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd $REPO_PATH/docs/tools +mkdir venv +virtualenv -p $(which python3) venv +source venv/bin/activate +python3 -m pip install --ignore-installed -r requirements.txt +./release.sh 2>&1 | tee tee $OUTPUT_PATH/output.log diff --git a/docker/images.json b/docker/images.json index 3e8adda868c..a6cc821108e 100644 --- a/docker/images.json +++ b/docker/images.json @@ -166,5 +166,20 @@ "docker/test/keeper-jepsen": { "name": "clickhouse/keeper-jepsen-test", "dependent": [] + }, + "docker/docs/builder": { + "name": "clickhouse/docs-builder", + "dependent": [ + "docker/docs/check", + "docker/docs/release" + ] + }, + "docker/docs/check": { + "name": "clickhouse/docs-check", + "dependent": [] + }, + "docker/docs/release": { + "name": "clickhouse/docs-release", + "dependent": [] } } diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 470dcc18233..d874ca422c3 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -193,8 +193,9 @@ if __name__ == "__main__": changed_images, dockerhub_repo_name = get_changed_docker_images(pr_info, repo_path, "docker/images.json") logging.info("Has changed images %s", ', '.join([str(image[0]) for image in changed_images])) pr_commit_version = str(pr_info.number) + '-' + pr_info.sha - versions = [str(pr_info.number), pr_commit_version] + if pr_info.number == 0: + versions.append("latest") subprocess.check_output("docker login --username 'robotclickhouse' --password '{}'".format(dockerhub_password), shell=True) diff --git a/tests/ci/docs_check.py b/tests/ci/docs_check.py new file mode 100644 index 00000000000..aece781a703 --- /dev/null +++ b/tests/ci/docs_check.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +import logging +import subprocess +import os +import time +import json +import sys +from github import Github +from report import create_test_html_report +from s3_helper import S3Helper +from pr_info import PRInfo +from get_robot_token import get_best_robot_token + +NAME = "Docs Check (actions)" + +def process_logs(s3_client, additional_logs, s3_path_prefix): + additional_urls = [] + for log_path in additional_logs: + if log_path: + additional_urls.append( + s3_client.upload_test_report_to_s3( + log_path, + s3_path_prefix + "/" + os.path.basename(log_path))) + + return additional_urls + +def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files): + s3_path_prefix = f"{pr_number}/{commit_sha}/docs_check" + additional_urls = process_logs(s3_client, additional_files, s3_path_prefix) + + branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master" + branch_name = "master" + if pr_number != 0: + branch_name = f"PR #{pr_number}" + branch_url = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_number}" + commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}" + + task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}" + + raw_log_url = additional_urls[0] + additional_urls.pop(0) + + html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls) + with open('report.html', 'w', encoding='utf-8') as f: + f.write(html_report) + + url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html") + logging.info("Search result in url %s", url) + return url + +def get_commit(gh, commit_sha): + repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse")) + commit = repo.get_commit(commit_sha) + return commit + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + temp_path = os.path.join(os.getenv("TEMP_PATH")) + repo_path = os.path.join(os.getenv("REPO_COPY")) + + with open(os.getenv('GITHUB_EVENT_PATH'), 'r', encoding='utf-8') as event_file: + event = json.load(event_file) + + pr_info = PRInfo(event, need_changed_files=True) + + gh = Github(get_best_robot_token()) + if not pr_info.has_changes_in_documentation(): + logging.info ("No changes in documentation") + commit = get_commit(gh, pr_info.sha) + commit.create_status(context=NAME, description="No changes in docs", state="success") + sys.exit(0) + + logging.info("Has changes in docs") + + if not os.path.exists(temp_path): + os.makedirs(temp_path) + + images_path = os.path.join(temp_path, 'changed_images.json') + + docker_image = 'clickhouse/docs-check' + if os.path.exists(images_path): + logging.info("Images file exists") + with open(images_path, 'r', encoding='utf-8') as images_fd: + images = json.load(images_fd) + logging.info("Got images %s", images) + if 'clickhouse/docs-check' in images: + docker_image += ':' + images['clickhouse/docs-check'] + + logging.info("Got docker image %s", docker_image) + for i in range(10): + try: + subprocess.check_output(f"docker pull {docker_image}", shell=True) + break + except Exception as ex: + time.sleep(i * 3) + logging.info("Got execption pulling docker %s", ex) + else: + raise Exception(f"Cannot pull dockerhub for image {docker_image}") + + test_output = os.path.join(temp_path, 'docs_check_log') + if not os.path.exists(test_output): + os.makedirs(test_output) + + cmd = f"docker run --cap-add=SYS_PTRACE --volume={repo_path}:/repo_path --volume={test_output}:/output_path {docker_image}" + + run_log_path = os.path.join(test_output, 'runlog.log') + + with open(run_log_path, 'w', encoding='utf-8') as log: + with subprocess.Popen(cmd, shell=True, stderr=log, stdout=log) as process: + retcode = process.wait() + if retcode == 0: + logging.info("Run successfully") + status = "success" + description = "Docs check passed" + else: + description = "Docs check failed (non zero exit code)" + status = "failure" + logging.info("Run failed") + + subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) + files = os.listdir(test_output) + lines = [] + additional_files = [] + if not files: + logging.error("No output files after docs check") + description = "No output files after docs check" + status = "failure" + else: + for f in files: + path = os.path.join(test_output, f) + additional_files.append(path) + with open(path, 'r', encoding='utf-8') as check_file: + for line in check_file: + if "ERROR" in line: + lines.append((line.split(':')[-1], "FAIL")) + if lines: + status = "failure" + description = "Found errors in docs" + elif status != "failure": + lines.append(("No errors found", "OK")) + else: + lines.append(("Non zero exit code", "FAIL")) + + s3_helper = S3Helper('https://s3.amazonaws.com') + + report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, lines, additional_files) + print("::notice ::Report url: {report_url}") + commit = get_commit(gh, pr_info.sha) + commit.create_status(context=NAME, description=description, state=status, target_url=report_url) diff --git a/tests/ci/docs_release.py b/tests/ci/docs_release.py new file mode 100644 index 00000000000..7ce7028fbf5 --- /dev/null +++ b/tests/ci/docs_release.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 + +#!/usr/bin/env python3 +import logging +import subprocess +import os +import time +import json +import sys +from github import Github +from report import create_test_html_report +from s3_helper import S3Helper +from pr_info import PRInfo +from get_robot_token import get_best_robot_token +from ssh import SSHKey + +NAME = "Docs Release (actions)" + +def process_logs(s3_client, additional_logs, s3_path_prefix): + additional_urls = [] + for log_path in additional_logs: + if log_path: + additional_urls.append( + s3_client.upload_test_report_to_s3( + log_path, + s3_path_prefix + "/" + os.path.basename(log_path))) + + return additional_urls + +def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files): + s3_path_prefix = f"{pr_number}/{commit_sha}/docs_release" + additional_urls = process_logs(s3_client, additional_files, s3_path_prefix) + + branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master" + branch_name = "master" + if pr_number != 0: + branch_name = f"PR #{pr_number}" + branch_url = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_number}" + commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}" + + task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}" + + raw_log_url = additional_urls[0] + additional_urls.pop(0) + + html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls) + with open('report.html', 'w', encoding='utf-8') as f: + f.write(html_report) + + url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html") + logging.info("Search result in url %s", url) + return url + +def get_commit(gh, commit_sha): + repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse")) + commit = repo.get_commit(commit_sha) + return commit + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + temp_path = os.path.join(os.getenv("TEMP_PATH")) + repo_path = os.path.join(os.getenv("REPO_COPY")) + + with open(os.getenv('GITHUB_EVENT_PATH'), 'r', encoding='utf-8') as event_file: + event = json.load(event_file) + + pr_info = PRInfo(event, need_changed_files=True) + + gh = Github(get_best_robot_token()) + if not pr_info.has_changes_in_documentation(): + logging.info ("No changes in documentation") + commit = get_commit(gh, pr_info.sha) + commit.create_status(context=NAME, description="No changes in docs", state="success") + sys.exit(0) + + logging.info("Has changes in docs") + + if not os.path.exists(temp_path): + os.makedirs(temp_path) + + images_path = os.path.join(temp_path, 'changed_images.json') + + docker_image = 'clickhouse/docs-release' + if os.path.exists(images_path): + logging.info("Images file exists") + with open(images_path, 'r', encoding='utf-8') as images_fd: + images = json.load(images_fd) + logging.info("Got images %s", images) + if 'clickhouse/docs-release' in images: + docker_image += ':' + images['clickhouse/docs-release'] + + logging.info("Got docker image %s", docker_image) + for i in range(10): + try: + subprocess.check_output(f"docker pull {docker_image}", shell=True) + break + except Exception as ex: + time.sleep(i * 3) + logging.info("Got execption pulling docker %s", ex) + else: + raise Exception(f"Cannot pull dockerhub for image {docker_image}") + + test_output = os.path.join(temp_path, 'docs_release_log') + if not os.path.exists(test_output): + os.makedirs(test_output) + + token = os.getenv('CLOUDFLARE_TOKEN') + cmd = f"docker run --cap-add=SYS_PTRACE -e CLOUDFLARE_TOKEN={token} --volume={repo_path}:/repo_path --volume={test_output}:/output_path {docker_image}" + + run_log_path = os.path.join(test_output, 'runlog.log') + + with open(run_log_path, 'w', encoding='utf-8') as log, SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"): + with subprocess.Popen(cmd, shell=True, stderr=log, stdout=log) as process: + retcode = process.wait() + if retcode == 0: + logging.info("Run successfully") + status = "success" + description = "Released successfuly" + else: + description = "Release failed (non zero exit code)" + status = "failure" + logging.info("Run failed") + + subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) + files = os.listdir(test_output) + lines = [] + additional_files = [] + if not files: + logging.error("No output files after docs release") + description = "No output files after docs release" + status = "failure" + else: + for f in files: + path = os.path.join(test_output, f) + additional_files.append(path) + with open(path, 'r', encoding='utf-8') as check_file: + for line in check_file: + if "ERROR" in line: + lines.append((line.split(':')[-1], "FAIL")) + if lines: + status = "failure" + description = "Found errors in docs" + elif status != "failure": + lines.append(("No errors found", "OK")) + else: + lines.append(("Non zero exit code", "FAIL")) + + s3_helper = S3Helper('https://s3.amazonaws.com') + + report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, lines, additional_files) + print("::notice ::Report url: {report_url}") + commit.create_status(context=NAME, description=description, state=status, target_url=report_url) diff --git a/tests/ci/metrics_lambda/app.py b/tests/ci/metrics_lambda/app.py index 8c7807e99cc..af0e0fe07f1 100644 --- a/tests/ci/metrics_lambda/app.py +++ b/tests/ci/metrics_lambda/app.py @@ -53,12 +53,21 @@ def list_runners(access_token): "Authorization": f"token {access_token}", "Accept": "application/vnd.github.v3+json", } - - response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners", headers=headers) + response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners?per_page=100", headers=headers) response.raise_for_status() data = response.json() - print("Total runners", data['total_count']) + total_runners = data['total_count'] runners = data['runners'] + + total_pages = int(total_runners / 100 + 1) + print("Total pages", total_pages) + for i in range(2, total_pages + 1): + response = requests.get(f"https://api.github.com/orgs/ClickHouse/actions/runners?page={i}&per_page=100", headers=headers) + response.raise_for_status() + data = response.json() + runners += data['runners'] + + print("Total runners", len(runners)) result = [] for runner in runners: tags = [tag['name'] for tag in runner['labels']] diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 0b4aeb56699..14a97e510a2 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -1,32 +1,50 @@ #!/usr/bin/env python3 +import os import urllib + import requests from unidiff import PatchSet +DIFF_IN_DOCUMENTATION_EXT = [".html", ".md", ".yml", ".txt", ".css", ".js", ".xml", ".ico", ".conf", ".svg", ".png", ".jpg", ".py", ".sh"] + class PRInfo: def __init__(self, github_event, need_orgs=False, need_changed_files=False): - self.number = github_event['number'] - if 'after' in github_event: + if 'pull_request' in github_event: # pull request and other similar events + self.number = github_event['number'] + if 'after' in github_event: + self.sha = github_event['after'] + else: + self.sha = github_event['pull_request']['head']['sha'] + + self.labels = { l['name'] for l in github_event['pull_request']['labels'] } + self.user_login = github_event['pull_request']['user']['login'] + self.user_orgs = set([]) + if need_orgs: + user_orgs_response = requests.get(github_event['pull_request']['user']['organizations_url']) + if user_orgs_response.ok: + response_json = user_orgs_response.json() + self.user_orgs = set(org['id'] for org in response_json) + + self.changed_files = set([]) + if need_changed_files: + diff_url = github_event['pull_request']['diff_url'] + diff = urllib.request.urlopen(diff_url) + diff_object = PatchSet(diff, diff.headers.get_charsets()[0]) + self.changed_files = { f.path for f in diff_object } + elif github_event['type'] == 'PushEvent': # push on master + self.number = 0 self.sha = github_event['after'] + self.labels = {} + if need_changed_files: + commit_before = github_event['before'] + diff = requests.get(f'https://api.github.com/repos/ClickHouse/ClickHouse/compare/{commit_before}...{self.sha}') + if 'files' in diff: + self.changed_files = [f['filename'] for f in diff['files']] + else: + self.changed_files = set([]) else: - self.sha = github_event['pull_request']['head']['sha'] - - self.labels = { l['name'] for l in github_event['pull_request']['labels'] } - self.user_login = github_event['pull_request']['user']['login'] - self.user_orgs = set([]) - if need_orgs: - user_orgs_response = requests.get(github_event['pull_request']['user']['organizations_url']) - if user_orgs_response.ok: - response_json = user_orgs_response.json() - self.user_orgs = set(org['id'] for org in response_json) - - self.changed_files = set([]) - if need_changed_files: - diff_url = github_event['pull_request']['diff_url'] - diff = urllib.request.urlopen(diff_url) - diff_object = PatchSet(diff, diff.headers.get_charsets()[0]) - self.changed_files = { f.path for f in diff_object } + raise Exception("Unknown event type") def get_dict(self): return { @@ -37,6 +55,18 @@ class PRInfo: 'user_orgs': self.user_orgs, } + def has_changes_in_documentation(self): + # If the list wasn't built yet the best we can do is to + # assume that there were changes. + if self.changed_files is None or not self.changed_files: + return True + + for f in self.changed_files: + _, ext = os.path.splitext(f) + if ext in DIFF_IN_DOCUMENTATION_EXT or 'Dockerfile' in f: + return True + return False + class FakePRInfo: def __init__(self): diff --git a/tests/ci/ssh.py b/tests/ci/ssh.py new file mode 100644 index 00000000000..1c0515364a8 --- /dev/null +++ b/tests/ci/ssh.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +import shutil +import os +import subprocess +import tempfile +import logging +import signal + + +class SSHAgent: + def __init__(self): + self._env = {} + self._env_backup = {} + self._keys = {} + self.start() + + @property + def pid(self): + return int(self._env["SSH_AGENT_PID"]) + + def start(self): + if shutil.which("ssh-agent") is None: + raise Exception("ssh-agent binary is not available") + + self._env_backup["SSH_AUTH_SOCK"] = os.environ.get("SSH_AUTH_SOCK") + self._env_backup["SSH_OPTIONS"] = os.environ.get("SSH_OPTIONS") + + # set ENV from stdout of ssh-agent + for line in self._run(['ssh-agent']).splitlines(): + name, _, value = line.partition(b"=") + if _ == b"=": + value = value.split(b";", 1)[0] + self._env[name.decode()] = value.decode() + os.environ[name.decode()] = value.decode() + + ssh_options = "," + os.environ["SSH_OPTIONS"] if os.environ.get("SSH_OPTIONS") else "" + os.environ["SSH_OPTIONS"] = f"{ssh_options}UserKnownHostsFile=/dev/null,StrictHostKeyChecking=no" + + def add(self, key): + key_pub = self._key_pub(key) + + if key_pub in self._keys: + self._keys[key_pub] += 1 + else: + self._run(["ssh-add", "-"], stdin=key.encode()) + self._keys[key_pub] = 1 + + return key_pub + + def remove(self, key_pub): + if key_pub not in self._keys: + raise Exception(f"Private key not found, public part: {key_pub}") + + if self._keys[key_pub] > 1: + self._keys[key_pub] -= 1 + else: + with tempfile.NamedTemporaryFile() as f: + f.write(key_pub) + f.flush() + self._run(["ssh-add", "-d", f.name]) + self._keys.pop(key_pub) + + def print_keys(self): + keys = self._run(["ssh-add", "-l"]).splitlines() + if keys: + logging.info("ssh-agent keys:") + for key in keys: + logging.info("%s", key) + else: + logging.info("ssh-agent (pid %d) is empty", self.pid) + + def kill(self): + for k, v in self._env.items(): + os.environ.pop(k, None) + + for k, v in self._env_backup.items(): + if v is not None: + os.environ[k] = v + + os.kill(self.pid, signal.SIGTERM) + + def _key_pub(self, key): + with tempfile.NamedTemporaryFile() as f: + f.write(key.encode()) + f.flush() + return self._run(["ssh-keygen", "-y", "-f", f.name]) + + @staticmethod + def _run(cmd, stdin=None): + shell = isinstance(cmd, str) + with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE if stdin else None, shell=shell) as p: + stdout, stderr = p.communicate(stdin) + + if stdout.strip().decode() == "The agent has no identities.": + return "" + + if p.returncode: + message = stderr.strip() + b"\n" + stdout.strip() + raise Exception(message.strip().decode()) + + return stdout + +class SSHKey: + def __init__(self, key_name): + self.key = os.getenv(key_name) + self._key_pub = None + self._ssh_agent = SSHAgent() + + def __enter__(self): + self._key_pub = self._ssh_agent.add(self.key) + self._ssh_agent.print_keys() + + def __exit__(self, exc_type, exc_val, exc_tb): + self._ssh_agent.remove(self._key_pub) + self._ssh_agent.print_keys() diff --git a/tests/ci/termination_lambda/app.py b/tests/ci/termination_lambda/app.py index 0b39cf73f25..cd7d51ae8eb 100644 --- a/tests/ci/termination_lambda/app.py +++ b/tests/ci/termination_lambda/app.py @@ -49,12 +49,20 @@ def list_runners(access_token): "Authorization": f"token {access_token}", "Accept": "application/vnd.github.v3+json", } - - response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners", headers=headers) + response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners?per_page=100", headers=headers) response.raise_for_status() data = response.json() - print("Total runners", data['total_count']) + total_runners = data['total_count'] runners = data['runners'] + + total_pages = int(total_runners / 100 + 1) + for i in range(2, total_pages + 1): + response = requests.get(f"https://api.github.com/orgs/ClickHouse/actions/runners?page={i}&per_page=100", headers=headers) + response.raise_for_status() + data = response.json() + runners += data['runners'] + + print("Total runners", len(runners)) result = [] for runner in runners: tags = [tag['name'] for tag in runner['labels']]