From e733e4ff745ae13dc7c01be6e1081909488aa8e5 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Wed, 18 Sep 2024 21:08:32 +0000
Subject: [PATCH] Backport #69557 to 24.3: Kill runner when integration tests
 fail to pre-pull

---
 tests/ci/ci_utils.py                 | 37 ++++++++++++++++++++++++++--
 tests/ci/integration_tests_runner.py |  5 +++-
 2 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index 97d42f9845b..e777e98cb5d 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -1,8 +1,13 @@
-from contextlib import contextmanager
+import logging
 import os
 import signal
-from typing import Any, List, Union, Iterator
+import subprocess
+import time
+from contextlib import contextmanager
 from pathlib import Path
+from typing import Any, Dict, Iterator, List, Tuple, Union
+
+logger = logging.getLogger(__name__)
 
 
 class WithIter(type):
@@ -51,6 +56,34 @@ class GHActions:
         print("::endgroup::")
 
 
+def kill_ci_runner(message: str) -> None:
+    """The function to kill the current process with all parents when it's possible.
+    Works only when run with the set `CI` environment"""
+    if not os.getenv("CI", ""):  # cycle import env_helper
+        logger.info("Running outside the CI, won't kill the runner")
+        return
+    print(f"::error::{message}")
+
+    def get_ppid_name(pid: int) -> Tuple[int, str]:
+        # Avoid using psutil, it's not in stdlib
+        stats = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8").split()
+        return int(stats[3]), stats[1]
+
+    pid = os.getpid()
+    pids = {}  # type: Dict[str, str]
+    while pid:
+        ppid, name = get_ppid_name(pid)
+        pids[str(pid)] = name
+        pid = ppid
+    logger.error(
+        "Sleeping 5 seconds and killing all possible processes from following:\n %s",
+        "\n ".join(f"{p}: {n}" for p, n in pids.items()),
+    )
+    time.sleep(5)
+    # The current process will be killed too
+    subprocess.run(f"kill -9 {' '.join(pids.keys())}", check=False, shell=True)
+
+
 def set_job_timeout():
     def timeout_handler(_signum, _frame):
         print("Timeout expired")
diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index 73613d6af87..26342d87e5e 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -18,6 +18,7 @@ from collections import defaultdict
 from itertools import chain
 from typing import Any, Dict
 
+from ci_utils import kill_ci_runner
 from env_helper import CI
 from integration_test_images import IMAGES
 
@@ -325,7 +326,9 @@ class ClickhouseIntegrationTestsRunner:
             except subprocess.CalledProcessError as err:
                 logging.info("docker-compose pull failed: %s", str(err))
                 continue
-        logging.error("Pulling images failed for 5 attempts. Will fail the worker.")
+        message = "Pulling images failed for 5 attempts. Will fail the worker."
+        logging.error(message)
+        kill_ci_runner(message)
         # We pass specific retcode to to ci/integration_test_check.py to skip status reporting and restart job
         sys.exit(13)