Merge pull request #61908 from ClickHouse/ci-fuzzer-enable

CI: enable libfuzzer (fixing build and docker)

.github/PULL_REQUEST_TEMPLATE.md

@@ -59,6 +59,9 @@ At a minimum, the following information should be added (but add more as needed)
 - [ ] <!---ci_exclude_tsan|msan|ubsan|coverage--> Exclude: All with TSAN, MSAN, UBSAN, Coverage
 - [ ] <!---ci_exclude_aarch64|release|debug--> Exclude: All with aarch64, release, debug
 ---
+- [ ] <!---ci_include_fuzzer--> Run only fuzzers related jobs (libFuzzer fuzzers, AST fuzzers, etc.)
+- [ ] <!---ci_exclude_ast--> Exclude: AST fuzzers
+---
 - [ ] <!---do_not_test--> Do not test
 - [ ] <!---woolen_wolfdog--> Woolen Wolfdog
 - [ ] <!---upload_all--> Upload binaries for special builds

base/base/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_compile_options($<$<OR:$<COMPILE_LANGUAGE:C>,$<COMPILE_LANGUAGE:CXX>>:${COVERAGE_FLAGS}>)
+add_compile_options("$<$<OR:$<COMPILE_LANGUAGE:C>,$<COMPILE_LANGUAGE:CXX>>:${COVERAGE_FLAGS}>")
 
 if (USE_CLANG_TIDY)
     set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}")

cmake/sanitize.cmake

@@ -57,8 +57,8 @@ option(WITH_COVERAGE "Instrumentation for code coverage with default implementat
 
 if (WITH_COVERAGE)
     message (STATUS "Enabled instrumentation for code coverage")
-    set(COVERAGE_FLAGS "SHELL:-fprofile-instr-generate -fcoverage-mapping")
-    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
+    set (COVERAGE_FLAGS -fprofile-instr-generate -fcoverage-mapping)
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
 endif()
 
 option (SANITIZE_COVERAGE "Instrumentation for code coverage with custom callbacks" OFF)

contrib/libprotobuf-mutator

@@ -1 +1 @@
-Subproject commit 1f95f8083066f5b38fd2db172e7e7f9aa7c49d2d
+Subproject commit b922c8ab9004ef9944982e4f165e2747b13223fa

docker/packager/binary-builder/build.sh

@@ -108,7 +108,8 @@ if [ -n "$MAKE_DEB" ]; then
   bash -x /build/packages/build
 fi
 
-mv ./programs/clickhouse* /output || mv ./programs/*_fuzzer /output
+mv ./programs/clickhouse* /output ||:
+mv ./programs/*_fuzzer /output ||:
 [ -x ./programs/self-extracting/clickhouse ] && mv ./programs/self-extracting/clickhouse /output
 [ -x ./programs/self-extracting/clickhouse-stripped ] && mv ./programs/self-extracting/clickhouse-stripped /output
 [ -x ./programs/self-extracting/clickhouse-keeper ] && mv ./programs/self-extracting/clickhouse-keeper /output

programs/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_compile_options($<$<OR:$<COMPILE_LANGUAGE:C>,$<COMPILE_LANGUAGE:CXX>>:${COVERAGE_FLAGS}>)
+add_compile_options("$<$<OR:$<COMPILE_LANGUAGE:C>,$<COMPILE_LANGUAGE:CXX>>:${COVERAGE_FLAGS}>")
 
 if (USE_CLANG_TIDY)
     set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}")

programs/server/fuzzers/tcp_protocol_fuzzer.cpp

@@ -10,6 +10,7 @@
 #include <Poco/Net/SocketAddress.h>
 #include <Poco/Net/StreamSocket.h>
 
+#include <Daemon/BaseDaemon.h>
 #include <Interpreters/Context.h>
 
 
@@ -25,6 +26,12 @@ static int64_t port = 9000;
 
 using namespace std::chrono_literals;
 
+void on_exit()
+{
+    BaseDaemon::terminate();
+    main_app.wait();
+}
+
 extern "C"
 int LLVMFuzzerInitialize(int * argc, char ***argv)
 {
@@ -60,6 +67,8 @@ int LLVMFuzzerInitialize(int * argc, char ***argv)
             exit(-1);
     }
 
+    atexit(on_exit);
+
     return 0;
 }
 

src/AggregateFunctions/fuzzers/CMakeLists.txt

@@ -1,2 +1,2 @@
 clickhouse_add_executable(aggregate_function_state_deserialization_fuzzer aggregate_function_state_deserialization_fuzzer.cpp ${SRCS})
-target_link_libraries(aggregate_function_state_deserialization_fuzzer PRIVATE dbms clickhouse_aggregate_functions clickhouse_functions)
+target_link_libraries(aggregate_function_state_deserialization_fuzzer PRIVATE clickhouse_functions clickhouse_aggregate_functions)

src/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_compile_options($<$<OR:$<COMPILE_LANGUAGE:C>,$<COMPILE_LANGUAGE:CXX>>:${COVERAGE_FLAGS}>)
+add_compile_options("$<$<OR:$<COMPILE_LANGUAGE:C>,$<COMPILE_LANGUAGE:CXX>>:${COVERAGE_FLAGS}>")
 
 if (USE_INCLUDE_WHAT_YOU_USE)
     set (CMAKE_CXX_INCLUDE_WHAT_YOU_USE ${IWYU_PATH})

src/Client/ClientBase.cpp

@@ -2751,7 +2751,7 @@ void ClientBase::runLibFuzzer()
     for (auto & arg : fuzzer_args_holder)
         fuzzer_args.emplace_back(arg.data());
 
-    int fuzzer_argc = fuzzer_args.size();
+    int fuzzer_argc = static_cast<int>(fuzzer_args.size());
     char ** fuzzer_argv = fuzzer_args.data();
 
     LLVMFuzzerRunDriver(&fuzzer_argc, &fuzzer_argv, [](const uint8_t * data, size_t size)

src/Core/fuzzers/CMakeLists.txt

@@ -1,2 +1,2 @@
 clickhouse_add_executable (names_and_types_fuzzer names_and_types_fuzzer.cpp)
-target_link_libraries (names_and_types_fuzzer PRIVATE dbms clickhouse_functions)
+target_link_libraries (names_and_types_fuzzer PRIVATE clickhouse_functions)

src/DataTypes/fuzzers/CMakeLists.txt

@@ -1,2 +1,2 @@
 clickhouse_add_executable(data_type_deserialization_fuzzer data_type_deserialization_fuzzer.cpp ${SRCS})
-target_link_libraries(data_type_deserialization_fuzzer PRIVATE dbms clickhouse_aggregate_functions clickhouse_functions)
+target_link_libraries(data_type_deserialization_fuzzer PRIVATE clickhouse_functions clickhouse_aggregate_functions)

src/Formats/fuzzers/CMakeLists.txt

@@ -1,2 +1,2 @@
 clickhouse_add_executable(format_fuzzer format_fuzzer.cpp ${SRCS})
-target_link_libraries(format_fuzzer PRIVATE dbms clickhouse_aggregate_functions clickhouse_functions)
+target_link_libraries(format_fuzzer PRIVATE clickhouse_functions clickhouse_aggregate_functions)

src/Formats/fuzzers/format_fuzzer.cpp

@@ -3,7 +3,6 @@
 #include <IO/ReadBufferFromMemory.h>
 #include <IO/ReadHelpers.h>
 
-#include <Formats/FormatFactory.h>
 #include <Formats/registerFormats.h>
 
 #include <QueryPipeline/Pipe.h>

src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt

@@ -39,7 +39,7 @@ set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 
 clickhouse_add_executable(codegen_select_fuzzer ${FUZZER_SRCS})
 
-set_source_files_properties("${PROTO_SRCS}" "out.cpp" PROPERTIES COMPILE_FLAGS "-Wno-reserved-identifier")
+set_source_files_properties("${PROTO_SRCS}" "out.cpp" PROPERTIES COMPILE_FLAGS "-Wno-reserved-identifier -Wno-extra-semi-stmt -Wno-used-but-marked-unused")
 
 # contrib/libprotobuf-mutator/src/libfuzzer/libfuzzer_macro.h:143:44: error: no newline at end of file [-Werror,-Wnewline-eof]
 target_compile_options (codegen_select_fuzzer PRIVATE -Wno-newline-eof)

src/Storages/fuzzers/CMakeLists.txt

@@ -4,4 +4,4 @@ clickhouse_add_executable (mergetree_checksum_fuzzer mergetree_checksum_fuzzer.c
 target_link_libraries (mergetree_checksum_fuzzer PRIVATE dbms clickhouse_functions)
 
 clickhouse_add_executable (columns_description_fuzzer columns_description_fuzzer.cpp)
-target_link_libraries (columns_description_fuzzer PRIVATE dbms clickhouse_functions)
+target_link_libraries (columns_description_fuzzer PRIVATE clickhouse_functions)

src/Storages/fuzzers/columns_description_fuzzer.cpp

@@ -1,4 +1,5 @@
 #include <Storages/ColumnsDescription.h>
+#include <iostream>
 
 #include <iostream>
 

tests/ci/libfuzzer_test_check.py

@@ -75,7 +75,7 @@ def get_run_command(
         f"--volume={result_path}:/test_output "
         "--security-opt seccomp=unconfined "  # required to issue io_uring sys-calls
         f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image} "
-        "python3 ./utils/runner.py"
+        "python3 /usr/share/clickhouse-test/fuzz/runner.py"
     )
 
 

tests/fuzz/runner.py

@@ -11,7 +11,7 @@ FUZZER_ARGS = os.getenv("FUZZER_ARGS", "")
 
 
 def run_fuzzer(fuzzer: str):
-    logging.info(f"Running fuzzer {fuzzer}...")
+    logging.info("Running fuzzer %s...", fuzzer)
 
     corpus_dir = f"{fuzzer}.in"
     with Path(corpus_dir) as path:
@@ -29,28 +29,27 @@ def run_fuzzer(fuzzer: str):
 
             if parser.has_section("asan"):
                 os.environ["ASAN_OPTIONS"] = (
-                    f"{os.environ['ASAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['asan'].items())}"
+                    f"{os.environ['ASAN_OPTIONS']}:{':'.join(f'{key}={value}' for key, value in parser['asan'].items())}"
                 )
 
             if parser.has_section("msan"):
                 os.environ["MSAN_OPTIONS"] = (
-                    f"{os.environ['MSAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['msan'].items())}"
+                    f"{os.environ['MSAN_OPTIONS']}:{':'.join(f'{key}={value}' for key, value in parser['msan'].items())}"
                 )
 
             if parser.has_section("ubsan"):
                 os.environ["UBSAN_OPTIONS"] = (
-                    f"{os.environ['UBSAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['ubsan'].items())}"
+                    f"{os.environ['UBSAN_OPTIONS']}:{':'.join(f'{key}={value}' for key, value in parser['ubsan'].items())}"
                 )
 
             if parser.has_section("libfuzzer"):
                 custom_libfuzzer_options = " ".join(
-                    "-%s=%s" % (key, value)
-                    for key, value in parser["libfuzzer"].items()
+                    f"-{key}={value}" for key, value in parser["libfuzzer"].items()
                 )
 
             if parser.has_section("fuzzer_arguments"):
                 fuzzer_arguments = " ".join(
-                    ("%s" % key) if value == "" else ("%s=%s" % (key, value))
+                    (f"{key}") if value == "" else (f"{key}={value}")
                     for key, value in parser["fuzzer_arguments"].items()
                 )
 
@@ -65,7 +64,7 @@ def run_fuzzer(fuzzer: str):
 
     cmd_line += " < /dev/null"
 
-    logging.info(f"...will execute: {cmd_line}")
+    logging.info("...will execute: %s", cmd_line)
     subprocess.check_call(cmd_line, shell=True)
 
 

tests/fuzz/tcp_protocol_fuzzer.options

@@ -0,0 +1,4 @@
+[fuzzer_arguments]
+--log-file=tcp_protocol_fuzzer.log
+--=
+--logging.terminal=0