From 97301a19b9e0a49ef4a84e8479bf03cfb6ec8d24 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 8 Dec 2021 18:51:53 +0100 Subject: [PATCH 1/2] Unify init scripts in one --- tests/ci/worker/init_builder.sh | 34 ---------------- tests/ci/worker/init_func_tester.sh | 34 ---------------- tests/ci/worker/init_fuzzer_unit_tester.sh | 34 ---------------- tests/ci/worker/init_runner.sh | 47 ++++++++++++++++++++++ tests/ci/worker/init_stress_tester.sh | 34 ---------------- tests/ci/worker/init_style_checker.sh | 20 --------- 6 files changed, 47 insertions(+), 156 deletions(-) delete mode 100644 tests/ci/worker/init_builder.sh delete mode 100644 tests/ci/worker/init_func_tester.sh delete mode 100644 tests/ci/worker/init_fuzzer_unit_tester.sh create mode 100644 tests/ci/worker/init_runner.sh delete mode 100644 tests/ci/worker/init_stress_tester.sh delete mode 100644 tests/ci/worker/init_style_checker.sh diff --git a/tests/ci/worker/init_builder.sh b/tests/ci/worker/init_builder.sh deleted file mode 100644 index 8fd00c1db0a..00000000000 --- a/tests/ci/worker/init_builder.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -set -uo pipefail - -echo "Running init script" -export DEBIAN_FRONTEND=noninteractive -export RUNNER_HOME=/home/ubuntu/actions-runner - -export RUNNER_URL="https://github.com/ClickHouse" -# Funny fact, but metadata service has fixed IP -export INSTANCE_ID=`curl -s http://169.254.169.254/latest/meta-data/instance-id` - -while true; do - runner_pid=`pgrep run.sh` - echo "Got runner pid $runner_pid" - - cd $RUNNER_HOME - if [ -z "$runner_pid" ]; then - echo "Receiving token" - RUNNER_TOKEN=`/usr/local/bin/aws ssm get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value` - - echo "Will try to remove runner" - sudo -u ubuntu ./config.sh remove --token $RUNNER_TOKEN ||: - - echo "Going to configure runner" - sudo -u ubuntu ./config.sh --url $RUNNER_URL --token $RUNNER_TOKEN --name $INSTANCE_ID --runnergroup Default --labels 'self-hosted,Linux,X64,builder' --work _work - - echo "Run" - sudo -u ubuntu ./run.sh & - sleep 15 - else - echo "Runner is working with pid $runner_pid, nothing to do" - sleep 10 - fi -done diff --git a/tests/ci/worker/init_func_tester.sh b/tests/ci/worker/init_func_tester.sh deleted file mode 100644 index d3ee3cb3d7f..00000000000 --- a/tests/ci/worker/init_func_tester.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -set -uo pipefail - -echo "Running init script" -export DEBIAN_FRONTEND=noninteractive -export RUNNER_HOME=/home/ubuntu/actions-runner - -export RUNNER_URL="https://github.com/ClickHouse" -# Funny fact, but metadata service has fixed IP -export INSTANCE_ID=`curl -s http://169.254.169.254/latest/meta-data/instance-id` - -while true; do - runner_pid=`pgrep run.sh` - echo "Got runner pid $runner_pid" - - cd $RUNNER_HOME - if [ -z "$runner_pid" ]; then - echo "Receiving token" - RUNNER_TOKEN=`/usr/local/bin/aws ssm get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value` - - echo "Will try to remove runner" - sudo -u ubuntu ./config.sh remove --token $RUNNER_TOKEN ||: - - echo "Going to configure runner" - sudo -u ubuntu ./config.sh --url $RUNNER_URL --token $RUNNER_TOKEN --name $INSTANCE_ID --runnergroup Default --labels 'self-hosted,Linux,X64,func-tester' --work _work - - echo "Run" - sudo -u ubuntu ./run.sh & - sleep 15 - else - echo "Runner is working with pid $runner_pid, nothing to do" - sleep 10 - fi -done diff --git a/tests/ci/worker/init_fuzzer_unit_tester.sh b/tests/ci/worker/init_fuzzer_unit_tester.sh deleted file mode 100644 index 2fbedba9e40..00000000000 --- a/tests/ci/worker/init_fuzzer_unit_tester.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -set -uo pipefail - -echo "Running init script" -export DEBIAN_FRONTEND=noninteractive -export RUNNER_HOME=/home/ubuntu/actions-runner - -export RUNNER_URL="https://github.com/ClickHouse" -# Funny fact, but metadata service has fixed IP -export INSTANCE_ID=`curl -s http://169.254.169.254/latest/meta-data/instance-id` - -while true; do - runner_pid=`pgrep run.sh` - echo "Got runner pid $runner_pid" - - cd $RUNNER_HOME - if [ -z "$runner_pid" ]; then - echo "Receiving token" - RUNNER_TOKEN=`/usr/local/bin/aws ssm get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value` - - echo "Will try to remove runner" - sudo -u ubuntu ./config.sh remove --token $RUNNER_TOKEN ||: - - echo "Going to configure runner" - sudo -u ubuntu ./config.sh --url $RUNNER_URL --token $RUNNER_TOKEN --name $INSTANCE_ID --runnergroup Default --labels 'self-hosted,Linux,X64,fuzzer-unit-tester' --work _work - - echo "Run" - sudo -u ubuntu ./run.sh & - sleep 15 - else - echo "Runner is working with pid $runner_pid, nothing to do" - sleep 10 - fi -done diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh new file mode 100644 index 00000000000..6838d925500 --- /dev/null +++ b/tests/ci/worker/init_runner.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -uo pipefail + +#################################### +# IMPORTANT! # +# EC2 instance should have # +# `github:runner-type` tag # +# set accordingly to a runner role # +#################################### + +echo "Running init script" +export DEBIAN_FRONTEND=noninteractive +export RUNNER_HOME=/home/ubuntu/actions-runner + +export RUNNER_URL="https://github.com/ClickHouse" +# Funny fact, but metadata service has fixed IP +INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) +export INSTANCE_ID + +# combine labels +RUNNER_TYPE=$(/usr/local/bin/aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" | jq '.Tags[] | select(."Key" == "github:runner-type") | .Value' -r) +LABELS="self-hosted,Linux,$(uname -m),$RUNNER_TYPE" +export LABELS + +while true; do + runner_pid=$(pgrep run.sh) + echo "Got runner pid $runner_pid" + + cd $RUNNER_HOME || exit 1 + if [ -z "$runner_pid" ]; then + echo "Receiving token" + RUNNER_TOKEN=$(/usr/local/bin/aws ssm get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value) + + echo "Will try to remove runner" + sudo -u ubuntu ./config.sh remove --token "$RUNNER_TOKEN" ||: + + echo "Going to configure runner" + sudo -u ubuntu ./config.sh --url $RUNNER_URL --token "$RUNNER_TOKEN" --name "$INSTANCE_ID" --runnergroup Default --labels "$LABELS" --work _work + + echo "Run" + sudo -u ubuntu ./run.sh & + sleep 15 + else + echo "Runner is working with pid $runner_pid, nothing to do" + sleep 10 + fi +done diff --git a/tests/ci/worker/init_stress_tester.sh b/tests/ci/worker/init_stress_tester.sh deleted file mode 100644 index 234f035e1ea..00000000000 --- a/tests/ci/worker/init_stress_tester.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -set -uo pipefail - -echo "Running init script" -export DEBIAN_FRONTEND=noninteractive -export RUNNER_HOME=/home/ubuntu/actions-runner - -export RUNNER_URL="https://github.com/ClickHouse" -# Funny fact, but metadata service has fixed IP -export INSTANCE_ID=`curl -s http://169.254.169.254/latest/meta-data/instance-id` - -while true; do - runner_pid=`pgrep run.sh` - echo "Got runner pid $runner_pid" - - cd $RUNNER_HOME - if [ -z "$runner_pid" ]; then - echo "Receiving token" - RUNNER_TOKEN=`/usr/local/bin/aws ssm get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value` - - echo "Will try to remove runner" - sudo -u ubuntu ./config.sh remove --token $RUNNER_TOKEN ||: - - echo "Going to configure runner" - sudo -u ubuntu ./config.sh --url $RUNNER_URL --token $RUNNER_TOKEN --name $INSTANCE_ID --runnergroup Default --labels 'self-hosted,Linux,X64,stress-tester' --work _work - - echo "Run" - sudo -u ubuntu ./run.sh & - sleep 15 - else - echo "Runner is working with pid $runner_pid, nothing to do" - sleep 10 - fi -done diff --git a/tests/ci/worker/init_style_checker.sh b/tests/ci/worker/init_style_checker.sh deleted file mode 100644 index 77cf66b5262..00000000000 --- a/tests/ci/worker/init_style_checker.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/bash -set -euo pipefail - -echo "Running init script" -export DEBIAN_FRONTEND=noninteractive -export RUNNER_HOME=/home/ubuntu/actions-runner - -echo "Receiving token" -export RUNNER_TOKEN=`/usr/local/bin/aws ssm get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value` -export RUNNER_URL="https://github.com/ClickHouse" -# Funny fact, but metadata service has fixed IP -export INSTANCE_ID=`curl -s http://169.254.169.254/latest/meta-data/instance-id` - -cd $RUNNER_HOME - -echo "Going to configure runner" -sudo -u ubuntu ./config.sh --url $RUNNER_URL --token $RUNNER_TOKEN --name $INSTANCE_ID --runnergroup Default --labels 'self-hosted,Linux,X64,style-checker' --work _work - -echo "Run" -sudo -u ubuntu ./run.sh From 546a03281f1c7911621c232d38ba1180ffaceb64 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 15 Dec 2021 11:10:47 +0100 Subject: [PATCH 2/2] Rework image building: - Make it arch-agnostic - Add necessary packages - Update runner version - Get public keys of ClickHouse/core members --- tests/ci/worker/ubuntu_style_check.sh | 49 ++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/tests/ci/worker/ubuntu_style_check.sh b/tests/ci/worker/ubuntu_style_check.sh index bf5c6057bed..585c8610772 100644 --- a/tests/ci/worker/ubuntu_style_check.sh +++ b/tests/ci/worker/ubuntu_style_check.sh @@ -1,25 +1,47 @@ #!/usr/bin/env bash -set -euo pipefail +set -xeuo pipefail echo "Running prepare script" export DEBIAN_FRONTEND=noninteractive -export RUNNER_VERSION=2.283.1 +export RUNNER_VERSION=2.285.1 export RUNNER_HOME=/home/ubuntu/actions-runner +deb_arch() { + case $(uname -m) in + x86_64 ) + echo amd64;; + aarch64 ) + echo arm64;; + esac +} + +runner_arch() { + case $(uname -m) in + x86_64 ) + echo x64;; + aarch64 ) + echo arm64;; + esac +} + apt-get update apt-get install --yes --no-install-recommends \ apt-transport-https \ + build-essential \ ca-certificates \ curl \ gnupg \ + jq \ lsb-release \ + pigz \ + python3-dev \ python3-pip \ unzip curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg -echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null +echo "deb [arch=$(deb_arch) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null apt-get update @@ -37,21 +59,32 @@ EOT systemctl restart docker -pip install boto3 pygithub requests urllib3 unidiff +pip install boto3 pygithub requests urllib3 unidiff dohq-artifactory mkdir -p $RUNNER_HOME && cd $RUNNER_HOME -curl -O -L https://github.com/actions/runner/releases/download/v$RUNNER_VERSION/actions-runner-linux-x64-$RUNNER_VERSION.tar.gz +RUNNER_ARCHIVE="actions-runner-linux-$(runner_arch)-$RUNNER_VERSION.tar.gz" -tar xzf ./actions-runner-linux-x64-$RUNNER_VERSION.tar.gz -rm -f ./actions-runner-linux-x64-$RUNNER_VERSION.tar.gz +curl -O -L "https://github.com/actions/runner/releases/download/v$RUNNER_VERSION/$RUNNER_ARCHIVE" + +tar xzf "./$RUNNER_ARCHIVE" +rm -f "./$RUNNER_ARCHIVE" ./bin/installdependencies.sh chown -R ubuntu:ubuntu $RUNNER_HOME cd /home/ubuntu -curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" unzip awscliv2.zip ./aws/install rm -rf /home/ubuntu/awscliv2.zip /home/ubuntu/aws + +# SSH keys of core team +mkdir -p /home/ubuntu/.ssh + +# ~/.ssh/authorized_keys is cleaned out, so we use deprecated but working ~/.ssh/authorized_keys2 +aws lambda invoke --region us-east-1 --function-name team-keys-lambda /tmp/core.keys +jq < /tmp/core.keys -r '.body' > /home/ubuntu/.ssh/authorized_keys2 +chown ubuntu: /home/ubuntu/.ssh -R +chmod 0700 /home/ubuntu/.ssh