Merge pull request #43579 from ClickHouse/universal-runners

Prepare CI for universal runners preallocated pool

tests/ci/ci_runners_metrics_lambda/app.py

@@ -11,6 +11,17 @@ import requests
 import boto3
 from botocore.exceptions import ClientError
 
+UNIVERSAL_LABEL = "universal"
+RUNNER_TYPE_LABELS = [
+    "builder",
+    "func-tester",
+    "func-tester-aarch64",
+    "fuzzer-unit-tester",
+    "stress-tester",
+    "style-checker",
+    "style-checker-aarch64",
+]
+
 
 def get_dead_runners_in_ec2(runners):
     ids = {
@@ -170,26 +181,23 @@ def list_runners(access_token):
 def group_runners_by_tag(listed_runners):
     result = {}
 
-    RUNNER_TYPE_LABELS = [
-        "builder",
-        "func-tester",
-        "func-tester-aarch64",
-        "fuzzer-unit-tester",
-        "stress-tester",
-        "style-checker",
-        "style-checker-aarch64",
-    ]
-    for runner in listed_runners:
-        for tag in runner.tags:
-            if tag in RUNNER_TYPE_LABELS:
+    def add_to_result(tag, runner):
         if tag not in result:
             result[tag] = []
         result[tag].append(runner)
+
+    for runner in listed_runners:
+        if UNIVERSAL_LABEL in runner.tags:
+            # Do not proceed other labels if UNIVERSAL_LABEL is included
+            add_to_result(UNIVERSAL_LABEL, runner)
+            continue
+
+        for tag in runner.tags:
+            if tag in RUNNER_TYPE_LABELS:
+                add_to_result(tag, runner)
                 break
         else:
-            if "unlabeled" not in result:
-                result["unlabeled"] = []
-            result["unlabeled"].append(runner)
+            add_to_result("unlabeled", runner)
     return result
 
 

tests/ci/ci_runners_metrics_lambda/build_and_deploy_archive.sh

@@ -0,0 +1 @@
+../team_keys_lambda/build_and_deploy_archive.sh

tests/ci/metrics_lambda/Dockerfile

@@ -1,13 +0,0 @@
-FROM public.ecr.aws/lambda/python:3.9
-
-# Install the function's dependencies using file requirements.txt
-# from your project folder.
-
-COPY requirements.txt  .
-RUN  pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
-
-# Copy function code
-COPY app.py ${LAMBDA_TASK_ROOT}
-
-# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
-CMD [ "app.handler" ]

tests/ci/team_keys_lambda/build_and_deploy_archive.sh

@@ -2,10 +2,13 @@
 set -xeo pipefail
 
 WORKDIR=$(dirname "$0")
+WORKDIR=$(readlink -f "${WORKDIR}")
 cd "$WORKDIR"
 
-PY_EXEC=python3.9
-LAMBDA_NAME=$(basename "$PWD")
+PY_VERSION=3.9
+PY_EXEC="python${PY_VERSION}"
+DOCKER_IMAGE="python:${PY_VERSION}-slim"
+LAMBDA_NAME=$(basename "$WORKDIR")
 LAMBDA_NAME=${LAMBDA_NAME//_/-}
 PACKAGE=lambda-package
 rm -rf "$PACKAGE" "$PACKAGE".zip
@@ -14,10 +17,12 @@ cp app.py "$PACKAGE"
 if [ -f requirements.txt ]; then
   VENV=lambda-venv
   rm -rf "$VENV" lambda-package.zip
-  "$PY_EXEC" -m venv "$VENV"
-  # shellcheck disable=SC1091
-  source "$VENV/bin/activate"
+  docker run --rm --user="${UID}" --volume="${WORKDIR}:/lambda" --workdir="/lambda" "${DOCKER_IMAGE}" \
+    /bin/bash -c "
+      '$PY_EXEC' -m venv '$VENV' &&
+      source '$VENV/bin/activate' &&
       pip install -r requirements.txt
+    "
   cp -rT "$VENV/lib/$PY_EXEC/site-packages/" "$PACKAGE"
   rm -r "$PACKAGE"/{pip,pip-*,setuptools,setuptools-*}
 fi

tests/ci/worker/init_runner.sh

@@ -46,15 +46,17 @@ curl "${TEAM_KEYS_URL}" > /home/ubuntu/.ssh/authorized_keys2
 chown ubuntu: /home/ubuntu/.ssh -R
 
 
-# Create a pre-run script that will restart docker daemon before the job started
+# Create a pre-run script that will provide diagnostics info
 mkdir -p /tmp/actions-hooks
-cat > /tmp/actions-hooks/pre-run.sh << 'EOF'
+cat > /tmp/actions-hooks/pre-run.sh << EOF
 #!/bin/bash
-set -xuo pipefail
+set -uo pipefail
 
 echo "Runner's public DNS: $(ec2metadata --public-hostname)"
+echo "Runner's labels: ${LABELS}"
 EOF
 
+# Create a post-run script that will restart docker daemon before the job started
 cat > /tmp/actions-hooks/post-run.sh << 'EOF'
 #!/bin/bash
 set -xuo pipefail