review suggestions

docker/images.json

@@ -21,6 +21,10 @@
         "name": "clickhouse/fuzzer",
         "dependent": []
     },
+    "docker/test/libfuzzer": {
+        "name": "clickhouse/libfuzzer",
+        "dependent": []
+    },
     "docker/test/performance-comparison": {
         "name": "clickhouse/performance-comparison",
         "dependent": []

docker/test/libfuzzer/Dockerfile

@@ -37,7 +37,7 @@ ENV FUZZER_ARGS="-max_total_time=60"
 
 SHELL ["/bin/bash", "-c"]
 CMD set -o pipefail \
-    && timeout -s 9 1h /run_libfuzzer.sh 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log
+    && timeout -s 9 1h /run_libfuzzer.py 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log
 
 # docker run --network=host --volume <workspace>:/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/libfuzzer
 

docker/test/libfuzzer/run_libfuzzer.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+import logging
+import os
+from pathlib import Path
+import subprocess
+from parse_options import parse_options
+
+DEBUGGER = os.getenv("DEBUGGER", "")
+FUZZER_ARGS = os.getenv("FUZZER_ARGS", "")
+
+
+def run_fuzzer(fuzzer: str):
+    logging.info(f"Running fuzzer {fuzzer}...")
+
+    corpus_dir = f"{fuzzer}.in"
+    with Path(corpus_dir) as path:
+        if not path.exists() or not path.is_dir():
+            corpus_dir = ""
+
+    options_file = f"{fuzzer}.options"
+    custom_libfuzzer_options = ""
+
+    with Path(options_file) as path:
+        if path.exists() and path.is_file():
+            custom_asan_options = parse_options(options_file, "asan")
+            if custom_asan_options:
+                os.environ[
+                    "ASAN_OPTIONS"
+                ] = f"{os.environ['ASAN_OPTIONS']}:{custom_asan_options}"
+
+            custom_msan_options = parse_options(options_file, "msan")
+            if custom_msan_options:
+                os.environ[
+                    "MSAN_OPTIONS"
+                ] = f"{os.environ['MSAN_OPTIONS']}:{custom_msan_options}"
+
+            custom_ubsan_options = parse_options(options_file, "ubsan")
+            if custom_ubsan_options:
+                os.environ[
+                    "UBSAN_OPTIONS"
+                ] = f"{os.environ['UBSAN_OPTIONS']}:{custom_ubsan_options}"
+
+            custom_libfuzzer_options = parse_options(options_file, "libfuzzer")
+
+    cmd_line = f"{DEBUGGER} ./{fuzzer} {FUZZER_ARGS} {corpus_dir}"
+    if custom_libfuzzer_options:
+        cmd_line += f" {custom_libfuzzer_options}"
+
+    if not "-dict=" in cmd_line and Path(f"{fuzzer}.dict").exists():
+        cmd_line += f" -dict={fuzzer}.dict"
+
+    cmd_line += " < /dev/null"
+
+    logging.info(f"...will execute: {cmd_line}")
+    subprocess.check_call(cmd_line, shell=True)
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+
+    subprocess.check_call("ls -al", shell=True)
+
+    with Path() as current:
+        for fuzzer in current.iterdir():
+            if (current / fuzzer).is_file() and os.access(current / fuzzer, os.X_OK):
+                run_fuzzer(fuzzer)
+
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

tests/ci/libfuzzer_test_check.py

@@ -30,7 +30,7 @@ from commit_status_helper import (
     # post_commit_status_to_file,
     update_mergeable_check,
 )
-from docker_pull_helper import DockerImage  # , get_image_with_version
+from docker_pull_helper import DockerImage, get_image_with_version
 
 # from download_release_packages import download_last_release
 from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH
@@ -329,10 +329,11 @@ def main():
     #             )
     #         sys.exit(0)
 
-    image_name = "clickhouse/libfuzzer-test"  # get_image_name(check_name)
-    docker_image = docker_build_image(
-        image_name, Path("../../docker/test/libfuzzer/")
-    )  # get_image_with_version(reports_path, image_name)
+    # image_name = "clickhouse/libfuzzer-test"  # get_image_name(check_name)
+    # docker_image = docker_build_image(
+    #     image_name, Path("../../docker/test/libfuzzer/")
+    # )
+    docker_image = get_image_with_version(reports_path, "clickhouse/libfuzzer")
 
     fuzzers_path = os.path.join(temp_path, "fuzzers")
     if not os.path.exists(fuzzers_path):