Preserve logs for failed installation tests, retry 3 times

tests/ci/install_check.py

@@ -19,6 +19,7 @@ from clickhouse_helper import (
     prepare_tests_results_for_clickhouse,
 )
 from commit_status_helper import post_commit_status, update_mergeable_check
+from compress_files import compress_fast
 from docker_pull_helper import get_image_with_version, DockerImage
 from env_helper import CI, TEMP_PATH as TEMP, REPORTS_PATH
 from get_robot_token import get_best_robot_token
@@ -34,6 +35,7 @@ from upload_result_helper import upload_results
 RPM_IMAGE = "clickhouse/install-rpm-test"
 DEB_IMAGE = "clickhouse/install-deb-test"
 TEMP_PATH = Path(TEMP)
+LOGS_PATH = TEMP_PATH / "tests_logs"
 SUCCESS = "success"
 FAILURE = "failure"
 OK = "OK"
@@ -42,9 +44,13 @@ FAIL = "FAIL"
 
 def prepare_test_scripts():
     server_test = r"""#!/bin/bash
+set -e
+trap "bash -ex /packages/preserve_logs.sh" ERR
 systemctl start clickhouse-server
 clickhouse-client -q 'SELECT version()'"""
     keeper_test = r"""#!/bin/bash
+set -e
+trap "bash -ex /packages/preserve_logs.sh" ERR
 systemctl start clickhouse-keeper
 for i in {1..20}; do
     echo wait for clickhouse-keeper to being up
@@ -61,6 +67,8 @@ for i in {1..5}; do
 done
 exec 13>&-"""
     binary_test = r"""#!/bin/bash
+set -e
+trap "bash -ex /packages/preserve_logs.sh" ERR
 /packages/clickhouse.copy install
 clickhouse-server start --daemon
 for i in {1..5}; do
@@ -81,9 +89,18 @@ for i in {1..5}; do
     exec 13>&-
 done
 exec 13>&-"""
+    preserve_logs = r"""#!/bin/bash
+journalctl -u clickhouse-server > /tests_logs/clickhouse-server.service || :
+journalctl -u clickhouse-keeper > /tests_logs/clickhouse-keeper.service || :
+cp /var/log/clickhouse-server/clickhouse-server.* /tests_logs/ || :
+cp /var/log/clickhouse-keeper/clickhouse-keeper.* /tests_logs/ || :
+chmod a+rw -R /tests_logs
+exit 1
+"""
     (TEMP_PATH / "server_test.sh").write_text(server_test, encoding="utf-8")
     (TEMP_PATH / "keeper_test.sh").write_text(keeper_test, encoding="utf-8")
     (TEMP_PATH / "binary_test.sh").write_text(binary_test, encoding="utf-8")
+    (TEMP_PATH / "preserve_logs.sh").write_text(preserve_logs, encoding="utf-8")
 
 
 def test_install_deb(image: DockerImage) -> TestResults:
@@ -148,27 +165,41 @@ def test_install(image: DockerImage, tests: Dict[str, str]) -> TestResults:
         stopwatch = Stopwatch()
         container_name = name.lower().replace(" ", "_").replace("/", "_")
         log_file = TEMP_PATH / f"{container_name}.log"
+        logs = [log_file]
         run_command = (
             f"docker run --rm --privileged --detach --cap-add=SYS_PTRACE "
-            f"--volume={TEMP_PATH}:/packages {image}"
+            f"--volume={LOGS_PATH}:/tests_logs --volume={TEMP_PATH}:/packages {image}"
         )
+
+        for retry in range(1, 4):
+            for file in LOGS_PATH.glob("*"):
+                file.unlink()
+
             logging.info("Running docker container: `%s`", run_command)
             container_id = subprocess.check_output(
                 run_command, shell=True, encoding="utf-8"
             ).strip()
             (TEMP_PATH / "install.sh").write_text(command)
-        install_command = f"docker exec {container_id} bash -ex /packages/install.sh"
+            install_command = (
+                f"docker exec {container_id} bash -ex /packages/install.sh"
+            )
             with TeePopen(install_command, log_file) as process:
                 retcode = process.wait()
                 if retcode == 0:
                     status = OK
-            else:
+                    break
+
                 status = FAIL
+            copy2(log_file, LOGS_PATH)
+            archive_path = TEMP_PATH / f"{container_name}-{retry}.tar.gz"
+            compress_fast(
+                LOGS_PATH.as_posix(),
+                archive_path.as_posix(),
+            )
+            logs.append(archive_path)
 
         subprocess.check_call(f"docker kill -s 9 {container_id}", shell=True)
-        test_results.append(
+        test_results.append(TestResult(name, status, stopwatch.duration_seconds, logs))
-            TestResult(name, status, stopwatch.duration_seconds, [log_file])
-        )
 
     return test_results
 
@@ -227,6 +258,7 @@ def main():
     args = parse_args()
 
     TEMP_PATH.mkdir(parents=True, exist_ok=True)
+    LOGS_PATH.mkdir(parents=True, exist_ok=True)
 
     pr_info = PRInfo()