Add timeouts to TeePopen, second attempt

tests/ci/fast_test_check.py

@@ -139,7 +139,7 @@ if __name__ == "__main__":
         os.makedirs(logs_path)
 
     run_log_path = os.path.join(logs_path, "runlog.log")
-    with TeePopen(run_cmd, run_log_path) as process:
+    with TeePopen(run_cmd, run_log_path, timeout=40 * 60) as process:
         retcode = process.wait()
         if retcode == 0:
             logging.info("Run successfully")

tests/ci/tee_popen.py

@@ -1,8 +1,11 @@
 #!/usr/bin/env python3
 
 from subprocess import Popen, PIPE, STDOUT
-import sys
+from threading import Thread
+from time import sleep
+import logging
 import os
+import sys
 
 
 # Very simple tee logic implementation. You can specify shell command, output
@@ -11,11 +14,23 @@ import os
 # stdout.
 class TeePopen:
     # pylint: disable=W0102
-    def __init__(self, command, log_file, env=os.environ.copy()):
+    def __init__(self, command, log_file, env=os.environ.copy(), timeout=None):
         self.command = command
         self.log_file = log_file
         self.env = env
         self.process = None
+        self.timeout = timeout
+
+    def _check_timeout(self):
+        sleep(self.timeout)
+        while self.process.poll() is None:
+            logging.warning(
+                "Killing process %s, timeout %s exceeded",
+                self.process.pid,
+                self.timeout,
+            )
+            os.killpg(self.process.pid, 9)
+            sleep(10)
 
     def __enter__(self):
         self.process = Popen(
@@ -23,11 +38,16 @@ class TeePopen:
             shell=True,
             universal_newlines=True,
             env=self.env,
+            start_new_session=True,  # signall will be sent to all children
             stderr=STDOUT,
             stdout=PIPE,
             bufsize=1,
         )
         self.log_file = open(self.log_file, "w", encoding="utf-8")
+        if self.timeout is not None and self.timeout > 0:
+            t = Thread(target=self._check_timeout)
+            t.daemon = True  # does not block the program from exit
+            t.start()
         return self
 
     def __exit__(self, t, value, traceback):