thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tune the shutting down on rebalance signal

Browse Source
This commit is contained in:
Mikhail f. Shiryaev 2023-06-07 13:30:48 +02:00
parent 026ab3d4ce
commit 915f431b95
No known key found for this signature in database
GPG Key ID: 4B02ED204C7D93F4
1 changed files with 14 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tests/ci/worker/init_runner.sh
View File

@ -71,8 +71,20 @@ check-terminating-metadata() {
# If there is a rebalance event, then the instance could die soon
# Let's don't wait for it and terminate proactively
if curl -s --fail http://169.254.169.254/latest/meta-data/events/recommendations/rebalance; then
echo 'The runner received rebalance recommendation, we are terminating'
terminate-and-exit
echo 'The received recommendation to rebalance, checking the uptime'
UPTIME=$(< /proc/uptime)
UPTIME=${UPTIME%%.*}
# We don't shutdown the instances younger than 30m
if (( 1800 < UPTIME )); then
# To not shutdown everything at once, use the 66% to survive
if (( $((RANDOM % 3)) == 0 )); then
echo 'The instance is older than 30m and won the roulette'
terminate-and-exit
fi
echo 'The instance is older than 30m, but is not chosen for rebalance'
else
echo 'The instance is younger than 30m, do not shut it down'
fi
fi
# Here we check if the autoscaling group marked the instance for termination, and it's wait for the job to finish