From 915f431b95cc0e762c8071e77c7c4a17560aeed5 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 7 Jun 2023 13:30:48 +0200 Subject: [PATCH] Tune the shutting down on rebalance signal --- tests/ci/worker/init_runner.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh index 23c1f3ab436..4190c3fca0b 100644 --- a/tests/ci/worker/init_runner.sh +++ b/tests/ci/worker/init_runner.sh @@ -71,8 +71,20 @@ check-terminating-metadata() { # If there is a rebalance event, then the instance could die soon # Let's don't wait for it and terminate proactively if curl -s --fail http://169.254.169.254/latest/meta-data/events/recommendations/rebalance; then - echo 'The runner received rebalance recommendation, we are terminating' - terminate-and-exit + echo 'The received recommendation to rebalance, checking the uptime' + UPTIME=$(< /proc/uptime) + UPTIME=${UPTIME%%.*} + # We don't shutdown the instances younger than 30m + if (( 1800 < UPTIME )); then + # To not shutdown everything at once, use the 66% to survive + if (( $((RANDOM % 3)) == 0 )); then + echo 'The instance is older than 30m and won the roulette' + terminate-and-exit + fi + echo 'The instance is older than 30m, but is not chosen for rebalance' + else + echo 'The instance is younger than 30m, do not shut it down' + fi fi # Here we check if the autoscaling group marked the instance for termination, and it's wait for the job to finish