Merge pull request #30839 from ClickHouse/qoega-docker-docs

Docs check on github actions

.github/workflows/main.yml

@@ -21,7 +21,6 @@ jobs:
           python3 run_check.py
   DockerHubPush:
     needs: CheckLabels
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Check out repository code
@@ -58,8 +57,34 @@ jobs:
           docker kill $(docker ps -q) ||:
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr $TEMP_PATH
+  DocsCheck:
+    needs: DockerHubPush
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ runner.temp }}/docs_check
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Docs Check
+        env:
+          TEMP_PATH: ${{runner.temp}}/docs_check
+          REPO_COPY: ${{runner.temp}}/docs_check/ClickHouse
+        run: |
+          cp -r $GITHUB_WORKSPACE $TEMP_PATH
+          cd $REPO_COPY/tests/ci
+          python3 docs_check.py
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill $(docker ps -q) ||:
+          docker rm -f $(docker ps -a -q) ||:
+          sudo rm -fr $TEMP_PATH
   BuilderDebDebug:
     needs: DockerHubPush
+    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
     runs-on: [self-hosted, builder]
     steps:
       - name: Download changed images
@@ -184,6 +209,7 @@ jobs:
           sudo rm -fr $TEMP_PATH
   FastTest:
     needs: DockerHubPush
+    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
     runs-on: [self-hosted, builder]
     steps:
       - name: Check out repository code

.github/workflows/release.yml

@@ -0,0 +1,50 @@
+name: ReleaseChecks
+concurrency:
+  group: master-release
+  cancel-in-progress: true
+on: # yamllint disable-line rule:truthy
+  push:
+    branches:
+      - master
+jobs:
+  DockerHubPush:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Images check
+        run: |
+          cd $GITHUB_WORKSPACE/tests/ci
+          python3 docker_images_check.py
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ runner.temp }}/docker_images_check/changed_images.json
+  DocsRelease:
+    needs: DockerHubPush
+    runs: [self-hosted, func-tester]
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{runner.temp}}/docs_release
+      - name: Docs Release
+        env:
+          TEMP_PATH: ${{runner.temp}}/docs_release
+          REPO_COPY: ${{runner.temp}}/docs_release/ClickHouse
+          CLOUDFLARE_TOKEN: ${{secrets.CLOUDFLARE}}
+          ROBOT_CLICKHOUSE_SSH_KEY: ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
+        run: |
+          cp -r $GITHUB_WORKSPACE $TEMP_PATH
+          cd $REPO_COPY/tests/ci
+          python3 docs_release.py
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill $(docker ps -q) ||:
+          docker rm -f $(docker ps -a -q) ||:
+          sudo rm -fr $TEMP_PATH

docker/docs/builder/Dockerfile

@@ -0,0 +1,43 @@
+# docker build -t clickhouse/docs-build .
+FROM ubuntu:20.04
+
+ENV LANG=C.UTF-8
+
+RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
+            python3-setuptools \
+            virtualenv \
+            wget \
+            bash \
+            python \
+            curl \
+            python3-requests \
+            sudo \
+            git \
+            openssl \
+            python3-pip \
+            software-properties-common \
+            language-pack-zh* \
+            chinese* \
+            fonts-arphic-ukai \
+            fonts-arphic-uming \
+            fonts-ipafont-mincho \
+            fonts-ipafont-gothic \
+            fonts-unfonts-core \
+            xvfb \
+            nodejs \
+            npm \
+            openjdk-11-jdk \
+            ssh-client \
+    && pip --no-cache-dir install scipy \
+    && apt-get autoremove --yes \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN wget 'https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb'
+
+RUN npm i -g purify-css
+
+RUN pip3 install --ignore-installed --upgrade setuptools pip virtualenv

docker/docs/check/Dockerfile

@@ -0,0 +1,9 @@
+# docker build -t clickhouse/docs-check .
+FROM clickhouse/docs-builder
+
+COPY run.sh /
+
+ENV REPO_PATH=/repo_path
+ENV OUTPUT_PATH=/output_path
+
+CMD ["/bin/bash", "/run.sh"]

docker/docs/check/run.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd $REPO_PATH/docs/tools
+mkdir venv
+virtualenv -p $(which python3) venv
+source venv/bin/activate
+python3 -m pip install --ignore-installed -r requirements.txt
+./build.py --skip-git-log 2>&1 | tee $OUTPUT_PATH/output.log

docker/docs/release/Dockerfile

@@ -0,0 +1,9 @@
+# docker build -t clickhouse/docs-release .
+FROM clickhouse/docs-builder
+
+COPY run.sh /
+
+ENV REPO_PATH=/repo_path
+ENV OUTPUT_PATH=/output_path
+
+CMD ["/bin/bash", "/run.sh"]

docker/docs/release/run.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd $REPO_PATH/docs/tools
+mkdir venv
+virtualenv -p $(which python3) venv
+source venv/bin/activate
+python3 -m pip install --ignore-installed -r requirements.txt
+./release.sh 2>&1 | tee tee $OUTPUT_PATH/output.log

docker/images.json

@@ -166,5 +166,20 @@
     "docker/test/keeper-jepsen": {
         "name": "clickhouse/keeper-jepsen-test",
         "dependent": []
+    },
+    "docker/docs/builder": {
+        "name": "clickhouse/docs-builder",
+        "dependent": [
+            "docker/docs/check",
+            "docker/docs/release"
+        ]
+    },
+    "docker/docs/check": {
+        "name": "clickhouse/docs-check",
+        "dependent": []
+    },
+    "docker/docs/release": {
+        "name": "clickhouse/docs-release",
+        "dependent": []
     }
 }

tests/ci/docker_images_check.py

@@ -193,8 +193,9 @@ if __name__ == "__main__":
     changed_images, dockerhub_repo_name = get_changed_docker_images(pr_info, repo_path, "docker/images.json")
     logging.info("Has changed images %s", ', '.join([str(image[0]) for image in changed_images]))
     pr_commit_version = str(pr_info.number) + '-' + pr_info.sha
-
     versions = [str(pr_info.number), pr_commit_version]
+    if pr_info.number == 0:
+        versions.append("latest")
 
     subprocess.check_output("docker login --username 'robotclickhouse' --password '{}'".format(dockerhub_password), shell=True)
 

tests/ci/docs_check.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+import logging
+import subprocess
+import os
+import time
+import json
+import sys
+from github import Github
+from report import create_test_html_report
+from s3_helper import S3Helper
+from pr_info import PRInfo
+from get_robot_token import get_best_robot_token
+
+NAME = "Docs Check (actions)"
+
+def process_logs(s3_client, additional_logs, s3_path_prefix):
+    additional_urls = []
+    for log_path in additional_logs:
+        if log_path:
+            additional_urls.append(
+                s3_client.upload_test_report_to_s3(
+                    log_path,
+                    s3_path_prefix + "/" + os.path.basename(log_path)))
+
+    return additional_urls
+
+def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files):
+    s3_path_prefix = f"{pr_number}/{commit_sha}/docs_check"
+    additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
+
+    branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
+    branch_name = "master"
+    if pr_number != 0:
+        branch_name = f"PR #{pr_number}"
+        branch_url = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_number}"
+    commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
+
+    task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
+
+    raw_log_url = additional_urls[0]
+    additional_urls.pop(0)
+
+    html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls)
+    with open('report.html', 'w', encoding='utf-8') as f:
+        f.write(html_report)
+
+    url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
+    logging.info("Search result in url %s", url)
+    return url
+
+def get_commit(gh, commit_sha):
+    repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
+    commit = repo.get_commit(commit_sha)
+    return commit
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    temp_path = os.path.join(os.getenv("TEMP_PATH"))
+    repo_path = os.path.join(os.getenv("REPO_COPY"))
+
+    with open(os.getenv('GITHUB_EVENT_PATH'), 'r', encoding='utf-8') as event_file:
+        event = json.load(event_file)
+
+    pr_info = PRInfo(event, need_changed_files=True)
+
+    gh = Github(get_best_robot_token())
+    if not pr_info.has_changes_in_documentation():
+        logging.info ("No changes in documentation")
+        commit = get_commit(gh, pr_info.sha)
+        commit.create_status(context=NAME, description="No changes in docs", state="success")
+        sys.exit(0)
+
+    logging.info("Has changes in docs")
+
+    if not os.path.exists(temp_path):
+        os.makedirs(temp_path)
+
+    images_path = os.path.join(temp_path, 'changed_images.json')
+
+    docker_image = 'clickhouse/docs-check'
+    if os.path.exists(images_path):
+        logging.info("Images file exists")
+        with open(images_path, 'r', encoding='utf-8') as images_fd:
+            images = json.load(images_fd)
+            logging.info("Got images %s", images)
+            if 'clickhouse/docs-check' in images:
+                docker_image += ':' + images['clickhouse/docs-check']
+
+    logging.info("Got docker image %s", docker_image)
+    for i in range(10):
+        try:
+            subprocess.check_output(f"docker pull {docker_image}", shell=True)
+            break
+        except Exception as ex:
+            time.sleep(i * 3)
+            logging.info("Got execption pulling docker %s", ex)
+    else:
+        raise Exception(f"Cannot pull dockerhub for image {docker_image}")
+
+    test_output = os.path.join(temp_path, 'docs_check_log')
+    if not os.path.exists(test_output):
+        os.makedirs(test_output)
+
+    cmd = f"docker run --cap-add=SYS_PTRACE --volume={repo_path}:/repo_path --volume={test_output}:/output_path {docker_image}"
+
+    run_log_path = os.path.join(test_output, 'runlog.log')
+
+    with open(run_log_path, 'w', encoding='utf-8') as log:
+        with subprocess.Popen(cmd, shell=True, stderr=log, stdout=log) as process:
+            retcode = process.wait()
+            if retcode == 0:
+                logging.info("Run successfully")
+                status = "success"
+                description = "Docs check passed"
+            else:
+                description = "Docs check failed (non zero exit code)"
+                status = "failure"
+                logging.info("Run failed")
+
+    subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
+    files = os.listdir(test_output)
+    lines = []
+    additional_files = []
+    if not files:
+        logging.error("No output files after docs check")
+        description = "No output files after docs check"
+        status = "failure"
+    else:
+        for f in files:
+            path = os.path.join(test_output, f)
+            additional_files.append(path)
+            with open(path, 'r', encoding='utf-8') as check_file:
+                for line in check_file:
+                    if "ERROR" in line:
+                        lines.append((line.split(':')[-1], "FAIL"))
+        if lines:
+            status = "failure"
+            description = "Found errors in docs"
+        elif status != "failure":
+            lines.append(("No errors found", "OK"))
+        else:
+            lines.append(("Non zero exit code", "FAIL"))
+
+    s3_helper = S3Helper('https://s3.amazonaws.com')
+
+    report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, lines, additional_files)
+    print("::notice ::Report url: {report_url}")
+    commit = get_commit(gh, pr_info.sha)
+    commit.create_status(context=NAME, description=description, state=status, target_url=report_url)

tests/ci/docs_release.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+
+#!/usr/bin/env python3
+import logging
+import subprocess
+import os
+import time
+import json
+import sys
+from github import Github
+from report import create_test_html_report
+from s3_helper import S3Helper
+from pr_info import PRInfo
+from get_robot_token import get_best_robot_token
+from ssh import SSHKey
+
+NAME = "Docs Release (actions)"
+
+def process_logs(s3_client, additional_logs, s3_path_prefix):
+    additional_urls = []
+    for log_path in additional_logs:
+        if log_path:
+            additional_urls.append(
+                s3_client.upload_test_report_to_s3(
+                    log_path,
+                    s3_path_prefix + "/" + os.path.basename(log_path)))
+
+    return additional_urls
+
+def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files):
+    s3_path_prefix = f"{pr_number}/{commit_sha}/docs_release"
+    additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
+
+    branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
+    branch_name = "master"
+    if pr_number != 0:
+        branch_name = f"PR #{pr_number}"
+        branch_url = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_number}"
+    commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
+
+    task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
+
+    raw_log_url = additional_urls[0]
+    additional_urls.pop(0)
+
+    html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls)
+    with open('report.html', 'w', encoding='utf-8') as f:
+        f.write(html_report)
+
+    url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
+    logging.info("Search result in url %s", url)
+    return url
+
+def get_commit(gh, commit_sha):
+    repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
+    commit = repo.get_commit(commit_sha)
+    return commit
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    temp_path = os.path.join(os.getenv("TEMP_PATH"))
+    repo_path = os.path.join(os.getenv("REPO_COPY"))
+
+    with open(os.getenv('GITHUB_EVENT_PATH'), 'r', encoding='utf-8') as event_file:
+        event = json.load(event_file)
+
+    pr_info = PRInfo(event, need_changed_files=True)
+
+    gh = Github(get_best_robot_token())
+    if not pr_info.has_changes_in_documentation():
+        logging.info ("No changes in documentation")
+        commit = get_commit(gh, pr_info.sha)
+        commit.create_status(context=NAME, description="No changes in docs", state="success")
+        sys.exit(0)
+
+    logging.info("Has changes in docs")
+
+    if not os.path.exists(temp_path):
+        os.makedirs(temp_path)
+
+    images_path = os.path.join(temp_path, 'changed_images.json')
+
+    docker_image = 'clickhouse/docs-release'
+    if os.path.exists(images_path):
+        logging.info("Images file exists")
+        with open(images_path, 'r', encoding='utf-8') as images_fd:
+            images = json.load(images_fd)
+            logging.info("Got images %s", images)
+            if 'clickhouse/docs-release' in images:
+                docker_image += ':' + images['clickhouse/docs-release']
+
+    logging.info("Got docker image %s", docker_image)
+    for i in range(10):
+        try:
+            subprocess.check_output(f"docker pull {docker_image}", shell=True)
+            break
+        except Exception as ex:
+            time.sleep(i * 3)
+            logging.info("Got execption pulling docker %s", ex)
+    else:
+        raise Exception(f"Cannot pull dockerhub for image {docker_image}")
+
+    test_output = os.path.join(temp_path, 'docs_release_log')
+    if not os.path.exists(test_output):
+        os.makedirs(test_output)
+
+    token = os.getenv('CLOUDFLARE_TOKEN')
+    cmd = f"docker run --cap-add=SYS_PTRACE -e CLOUDFLARE_TOKEN={token} --volume={repo_path}:/repo_path --volume={test_output}:/output_path {docker_image}"
+
+    run_log_path = os.path.join(test_output, 'runlog.log')
+
+    with open(run_log_path, 'w', encoding='utf-8') as log, SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"):
+        with subprocess.Popen(cmd, shell=True, stderr=log, stdout=log) as process:
+            retcode = process.wait()
+            if retcode == 0:
+                logging.info("Run successfully")
+                status = "success"
+                description = "Released successfuly"
+            else:
+                description = "Release failed (non zero exit code)"
+                status = "failure"
+                logging.info("Run failed")
+
+    subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
+    files = os.listdir(test_output)
+    lines = []
+    additional_files = []
+    if not files:
+        logging.error("No output files after docs release")
+        description = "No output files after docs release"
+        status = "failure"
+    else:
+        for f in files:
+            path = os.path.join(test_output, f)
+            additional_files.append(path)
+            with open(path, 'r', encoding='utf-8') as check_file:
+                for line in check_file:
+                    if "ERROR" in line:
+                        lines.append((line.split(':')[-1], "FAIL"))
+        if lines:
+            status = "failure"
+            description = "Found errors in docs"
+        elif status != "failure":
+            lines.append(("No errors found", "OK"))
+        else:
+            lines.append(("Non zero exit code", "FAIL"))
+
+    s3_helper = S3Helper('https://s3.amazonaws.com')
+
+    report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, lines, additional_files)
+    print("::notice ::Report url: {report_url}")
+    commit.create_status(context=NAME, description=description, state=status, target_url=report_url)

tests/ci/metrics_lambda/app.py

@@ -53,12 +53,21 @@ def list_runners(access_token):
         "Authorization": f"token {access_token}",
         "Accept": "application/vnd.github.v3+json",
     }
-
-    response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners", headers=headers)
+    response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners?per_page=100", headers=headers)
     response.raise_for_status()
     data = response.json()
-    print("Total runners", data['total_count'])
+    total_runners = data['total_count']
     runners = data['runners']
+
+    total_pages = int(total_runners / 100 + 1)
+    print("Total pages", total_pages)
+    for i in range(2, total_pages + 1):
+        response = requests.get(f"https://api.github.com/orgs/ClickHouse/actions/runners?page={i}&per_page=100", headers=headers)
+        response.raise_for_status()
+        data = response.json()
+        runners += data['runners']
+
+    print("Total runners", len(runners))
     result = []
     for runner in runners:
         tags = [tag['name'] for tag in runner['labels']]

tests/ci/pr_info.py

@@ -1,32 +1,50 @@
 #!/usr/bin/env python3
+import os
 import urllib
+
 import requests
 from unidiff import PatchSet
 
 
+DIFF_IN_DOCUMENTATION_EXT = [".html", ".md", ".yml", ".txt", ".css", ".js", ".xml", ".ico", ".conf", ".svg", ".png", ".jpg", ".py", ".sh"]
+
 class PRInfo:
     def __init__(self, github_event, need_orgs=False, need_changed_files=False):
-        self.number = github_event['number']
-        if 'after' in github_event:
+        if 'pull_request' in github_event: # pull request and other similar events
+            self.number = github_event['number']
+            if 'after' in github_event:
+                self.sha = github_event['after']
+            else:
+                self.sha = github_event['pull_request']['head']['sha']
+
+            self.labels = { l['name'] for l in github_event['pull_request']['labels'] }
+            self.user_login = github_event['pull_request']['user']['login']
+            self.user_orgs = set([])
+            if need_orgs:
+                user_orgs_response = requests.get(github_event['pull_request']['user']['organizations_url'])
+                if user_orgs_response.ok:
+                    response_json = user_orgs_response.json()
+                    self.user_orgs = set(org['id'] for org in response_json)
+
+            self.changed_files = set([])
+            if need_changed_files:
+                diff_url = github_event['pull_request']['diff_url']
+                diff = urllib.request.urlopen(diff_url)
+                diff_object = PatchSet(diff, diff.headers.get_charsets()[0])
+                self.changed_files = { f.path for f in diff_object }
+        elif github_event['type'] == 'PushEvent': # push on master
+            self.number = 0
             self.sha = github_event['after']
+            self.labels = {}
+            if need_changed_files:
+                commit_before = github_event['before']
+                diff = requests.get(f'https://api.github.com/repos/ClickHouse/ClickHouse/compare/{commit_before}...{self.sha}')
+                if 'files' in diff:
+                    self.changed_files = [f['filename'] for f in diff['files']]
+                else:
+                    self.changed_files = set([])
         else:
-            self.sha = github_event['pull_request']['head']['sha']
-
-        self.labels = { l['name'] for l in github_event['pull_request']['labels'] }
-        self.user_login = github_event['pull_request']['user']['login']
-        self.user_orgs = set([])
-        if need_orgs:
-            user_orgs_response = requests.get(github_event['pull_request']['user']['organizations_url'])
-            if user_orgs_response.ok:
-                response_json = user_orgs_response.json()
-                self.user_orgs = set(org['id'] for org in response_json)
-
-        self.changed_files = set([])
-        if need_changed_files:
-            diff_url = github_event['pull_request']['diff_url']
-            diff = urllib.request.urlopen(diff_url)
-            diff_object = PatchSet(diff, diff.headers.get_charsets()[0])
-            self.changed_files = { f.path for f in diff_object }
+            raise Exception("Unknown event type")
 
     def get_dict(self):
         return {
@@ -37,6 +55,18 @@ class PRInfo:
             'user_orgs': self.user_orgs,
         }
 
+    def has_changes_in_documentation(self):
+        # If the list wasn't built yet the best we can do is to
+        # assume that there were changes.
+        if self.changed_files is None or not self.changed_files:
+            return True
+
+        for f in self.changed_files:
+            _, ext = os.path.splitext(f)
+            if ext in DIFF_IN_DOCUMENTATION_EXT or 'Dockerfile' in f:
+                return True
+        return False
+
 
 class FakePRInfo:
     def __init__(self):

tests/ci/ssh.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+import shutil
+import os
+import subprocess
+import tempfile
+import logging
+import signal
+
+
+class SSHAgent:
+    def __init__(self):
+        self._env = {}
+        self._env_backup = {}
+        self._keys = {}
+        self.start()
+
+    @property
+    def pid(self):
+        return int(self._env["SSH_AGENT_PID"])
+
+    def start(self):
+        if shutil.which("ssh-agent") is None:
+            raise Exception("ssh-agent binary is not available")
+
+        self._env_backup["SSH_AUTH_SOCK"] = os.environ.get("SSH_AUTH_SOCK")
+        self._env_backup["SSH_OPTIONS"] = os.environ.get("SSH_OPTIONS")
+
+        # set ENV from stdout of ssh-agent
+        for line in self._run(['ssh-agent']).splitlines():
+            name, _, value = line.partition(b"=")
+            if _ == b"=":
+                value = value.split(b";", 1)[0]
+                self._env[name.decode()] = value.decode()
+                os.environ[name.decode()] = value.decode()
+
+        ssh_options = "," + os.environ["SSH_OPTIONS"] if os.environ.get("SSH_OPTIONS") else ""
+        os.environ["SSH_OPTIONS"] = f"{ssh_options}UserKnownHostsFile=/dev/null,StrictHostKeyChecking=no"
+
+    def add(self, key):
+        key_pub = self._key_pub(key)
+
+        if key_pub in self._keys:
+            self._keys[key_pub] += 1
+        else:
+            self._run(["ssh-add", "-"], stdin=key.encode())
+            self._keys[key_pub] = 1
+
+        return key_pub
+
+    def remove(self, key_pub):
+        if key_pub not in self._keys:
+            raise Exception(f"Private key not found, public part: {key_pub}")
+
+        if self._keys[key_pub] > 1:
+            self._keys[key_pub] -= 1
+        else:
+            with tempfile.NamedTemporaryFile() as f:
+                f.write(key_pub)
+                f.flush()
+                self._run(["ssh-add", "-d", f.name])
+            self._keys.pop(key_pub)
+
+    def print_keys(self):
+        keys = self._run(["ssh-add", "-l"]).splitlines()
+        if keys:
+            logging.info("ssh-agent keys:")
+            for key in keys:
+                logging.info("%s", key)
+        else:
+            logging.info("ssh-agent (pid %d) is empty", self.pid)
+
+    def kill(self):
+        for k, v in self._env.items():
+            os.environ.pop(k, None)
+
+        for k, v in self._env_backup.items():
+            if v is not None:
+                os.environ[k] = v
+
+        os.kill(self.pid, signal.SIGTERM)
+
+    def _key_pub(self, key):
+        with tempfile.NamedTemporaryFile() as f:
+            f.write(key.encode())
+            f.flush()
+            return self._run(["ssh-keygen", "-y", "-f", f.name])
+
+    @staticmethod
+    def _run(cmd, stdin=None):
+        shell = isinstance(cmd, str)
+        with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE if stdin else None, shell=shell) as p:
+            stdout, stderr = p.communicate(stdin)
+
+            if stdout.strip().decode() == "The agent has no identities.":
+                return ""
+
+            if p.returncode:
+                message = stderr.strip() + b"\n" + stdout.strip()
+                raise Exception(message.strip().decode())
+
+            return stdout
+
+class SSHKey:
+    def __init__(self, key_name):
+        self.key = os.getenv(key_name)
+        self._key_pub = None
+        self._ssh_agent = SSHAgent()
+
+    def __enter__(self):
+        self._key_pub = self._ssh_agent.add(self.key)
+        self._ssh_agent.print_keys()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._ssh_agent.remove(self._key_pub)
+        self._ssh_agent.print_keys()

tests/ci/termination_lambda/app.py

@@ -49,12 +49,20 @@ def list_runners(access_token):
         "Authorization": f"token {access_token}",
         "Accept": "application/vnd.github.v3+json",
     }
-
-    response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners", headers=headers)
+    response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners?per_page=100", headers=headers)
     response.raise_for_status()
     data = response.json()
-    print("Total runners", data['total_count'])
+    total_runners = data['total_count']
     runners = data['runners']
+
+    total_pages = int(total_runners / 100 + 1)
+    for i in range(2, total_pages + 1):
+        response = requests.get(f"https://api.github.com/orgs/ClickHouse/actions/runners?page={i}&per_page=100", headers=headers)
+        response.raise_for_status()
+        data = response.json()
+        runners += data['runners']
+
+    print("Total runners", len(runners))
     result = []
     for runner in runners:
         tags = [tag['name'] for tag in runner['labels']]