Merge pull request #27990 from nikitamikhaylov/fix-clang-tidy

Build fuzzers in CI

CMakeLists.txt

@@ -127,12 +127,13 @@ if (USE_STATIC_LIBRARIES)
     list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES)
 endif ()
 
-# Implies ${WITH_COVERAGE}
 option (ENABLE_FUZZING "Fuzzy testing using libfuzzer" OFF)
 
 if (ENABLE_FUZZING)
+    # Also set WITH_COVERAGE=1 for better fuzzing process
+    # By default this is disabled, because fuzzers are built in CI with the clickhouse itself.
+    # And we don't want to enable coverage for it.
     message (STATUS "Fuzzing instrumentation enabled")
-    set (WITH_COVERAGE ON)
     set (FUZZER "libfuzzer")
 endif()
 

docker/packager/binary/build.sh

@@ -83,6 +83,16 @@ then
     mv "$COMBINED_OUTPUT.tgz" /output
 fi
 
+# Also build fuzzers if any sanitizer specified
+if [ -n "$SANITIZER" ]
+then
+  # Currently we are in build/build_docker directory
+  ../docker/packager/other/fuzzer.sh
+fi
+
+ccache --show-config ||:
+ccache --show-stats ||:
+
 if [ "${CCACHE_DEBUG:-}" == "1" ]
 then
     find . -name '*.ccache-*' -print0 \
@@ -95,4 +105,3 @@ then
     # files in place, and will fail because this directory is not writable.
     tar -cv -I pixz -f /output/ccache.log.txz "$CCACHE_LOGFILE"
 fi
-

docker/packager/deb/build.sh

@@ -23,12 +23,24 @@ then
   echo "Place $BINARY_OUTPUT to output"
   mkdir /output/binary ||: # if exists
   mv /build/obj-*/programs/clickhouse* /output/binary
+
   if [ "$BINARY_OUTPUT" = "tests" ]
   then
     mv /build/obj-*/src/unit_tests_dbms /output/binary
   fi
 fi
 
+# Also build fuzzers if any sanitizer specified
+if [ -n "$SANITIZER" ]
+then
+  # Script is supposed that we are in build directory.
+  mkdir -p build/build_docker
+  cd build/build_docker
+  # Launching build script
+  ../docker/packager/other/fuzzer.sh
+  cd
+fi
+
 ccache --show-config ||:
 ccache --show-stats ||:
 

docker/packager/other/fuzzer.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# This script is responsible for building all fuzzers, and copy them to output directory
+# as an archive.
+# Script is supposed that we are in build directory.
+
+set -x -e
+
+printenv
+
+# Delete previous cache, because we add a new flags -DENABLE_FUZZING=1 and -DFUZZER=libfuzzer
+rm -f CMakeCache.txt
+read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}"
+# Hope, that the most part of files will be in cache, so we just link new executables
+cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" -DENABLE_CLICKHOUSE_ODBC_BRIDGE=OFF \
+    -DENABLE_LIBRARIES=0 -DENABLE_SSL=1 -DUSE_INTERNAL_SSL_LIBRARY=1 -DUSE_UNWIND=ON -DENABLE_EMBEDDED_COMPILER=0 \
+    -DENABLE_EXAMPLES=0 -DENABLE_UTILS=0 -DENABLE_THINLTO=0 "-DSANITIZE=$SANITIZER" \
+    -DENABLE_FUZZING=1 -DFUZZER='libfuzzer' -DENABLE_TCMALLOC=0 -DENABLE_JEMALLOC=0 \
+    -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
+
+FUZZER_TARGETS=$(find ../src -name '*_fuzzer.cpp' -execdir basename {} .cpp ';' | tr '\n' ' ')
+
+mkdir -p /output/fuzzers
+for FUZZER_TARGET in $FUZZER_TARGETS
+do
+    # shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
+    ninja $NINJA_FLAGS $FUZZER_TARGET
+    # Find this binary in build directory and strip it
+    FUZZER_PATH=$(find ./src -name "$FUZZER_TARGET")
+    strip --strip-unneeded "$FUZZER_PATH"
+    mv "$FUZZER_PATH" /output/fuzzers
+done
+
+tar -zcvf /output/fuzzers.tar.gz /output/fuzzers
+rm -rf /output/fuzzers

docker/packager/packager

@@ -105,6 +105,9 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
     if image_type == "deb" or image_type == "unbundled":
         result.append("DEB_CC={}".format(cc))
         result.append("DEB_CXX={}".format(cxx))
+        # For building fuzzers
+        result.append("CC={}".format(cc))
+        result.append("CXX={}".format(cxx))
     elif image_type == "binary":
         result.append("CC={}".format(cc))
         result.append("CXX={}".format(cxx))

release

@@ -60,9 +60,6 @@ then
     elif [[ "$SANITIZER" == "thread" ]]; then VERSION_POSTFIX+="+tsan"
     elif [[ "$SANITIZER" == "memory" ]]; then VERSION_POSTFIX+="+msan"
     elif [[ "$SANITIZER" == "undefined" ]]; then VERSION_POSTFIX+="+ubsan"
-    elif [[ "$SANITIZER" == "libfuzzer" ]]; then
-        VERSION_POSTFIX+="+libfuzzer"
-        MALLOC_OPTS="-DENABLE_TCMALLOC=0 -DENABLE_JEMALLOC=0"
     else
         echo "Unknown value of SANITIZER variable: $SANITIZER"
         exit 3

src/Compression/fuzzers/CMakeLists.txt

@@ -1,2 +1,8 @@
 add_executable (compressed_buffer_fuzzer compressed_buffer_fuzzer.cpp)
-target_link_libraries (compressed_buffer_fuzzer PRIVATE fuzz_compression clickhouse_common_io ${LIB_FUZZING_ENGINE})
+
+# Our code has strong cohesion and target associated with `Compression` also depends on `DataTypes`.
+# But we can exclude some files which have dependencies in case of
+# fuzzer related build (we are interested in fuzzing only particular part of our code).
+# So, some symbols will be declared, but not defined. Unfortunately, this trick doesn't work with UBSan.
+# If you want really small size of the resulted binary, just link with fuzz_compression and clickhouse_common_io
+target_link_libraries (compressed_buffer_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE})

src/Storages/fuzzers/CMakeLists.txt

@@ -1,11 +1,8 @@
 
-add_executable (mergetree_checksum_fuzzer
-    mergetree_checksum_fuzzer.cpp
-    "${ClickHouse_SOURCE_DIR}/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp"
-    "${ClickHouse_SOURCE_DIR}/src/Compression/CompressedReadBuffer.cpp"
-    "${ClickHouse_SOURCE_DIR}/src/Compression/CompressedWriteBuffer.cpp"
-)
-target_link_libraries (mergetree_checksum_fuzzer PRIVATE clickhouse_common_io fuzz_compression ${LIB_FUZZING_ENGINE})
+add_executable (mergetree_checksum_fuzzer mergetree_checksum_fuzzer.cpp)
+
+# Look at comment around fuzz_compression target declaration
+target_link_libraries (mergetree_checksum_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE})
 
 add_executable (columns_description_fuzzer columns_description_fuzzer.cpp)
 target_link_libraries (columns_description_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE})