From 242ef3dcb5d5f2d9f7f3d5c40e265b93146cd435 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 24 Aug 2021 00:09:19 +0000 Subject: [PATCH] Proper build --- docker/packager/binary/build.sh | 37 +++++++++------------------------ docker/packager/deb/build.sh | 25 ++++++---------------- docker/packager/other/fuzzer.sh | 30 ++++++++++++++++++++++++++ release | 3 --- 4 files changed, 46 insertions(+), 49 deletions(-) create mode 100644 docker/packager/other/fuzzer.sh diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 9a77d3b2fe3..b9900e34bf1 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -83,6 +83,16 @@ then mv "$COMBINED_OUTPUT.tgz" /output fi +# Also build fuzzers if any sanitizer specified +if [ -n "$SANITIZER" ] +then + # Currently we are in build/build_docker directory + ../docker/packager/other/fuzzer.sh +fi + +ccache --show-config ||: +ccache --show-stats ||: + if [ "${CCACHE_DEBUG:-}" == "1" ] then find . -name '*.ccache-*' -print0 \ @@ -95,30 +105,3 @@ then # files in place, and will fail because this directory is not writable. tar -cv -I pixz -f /output/ccache.log.txz "$CCACHE_LOGFILE" fi - -# Also build fuzzers if any sanitizer specified -if [ -n "$SANITIZER" ] -then - # Delete previous cache, because we add a new flag -DENABLE_FUZZING=1 - rm -rf CMakeCache.txt CMakeFiles/ - - # Hope, that the most part of files will be in cache, so we just link new executables - cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" \ - "-DSANITIZE=$SANITIZER" -DENABLE_FUZZING=1 -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" .. - - FUZZER_TARGETS=$(find ../src -name '*_fuzzer.cpp' -execdir basename {} .cpp ';' | tr '\n' ' ') - - mkdir -p /output/fuzzers - for FUZZER_TARGET in $FUZZER_TARGETS - do - # shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty. - ninja $NINJA_FLAGS $FUZZER_TARGET - # Find this binary in build directory and strip it - FUZZER_PATH=$(find ./src -name "$FUZZER_TARGET") - strip --strip-unneeded "$FUZZER_PATH" - mv "$FUZZER_PATH" /output/fuzzers - done - - tar -zcvf /output/fuzzers.tar.gz /output/fuzzers - rm -rf /output/fuzzers -fi diff --git a/docker/packager/deb/build.sh b/docker/packager/deb/build.sh index d343287eee9..5016e40dd2e 100755 --- a/docker/packager/deb/build.sh +++ b/docker/packager/deb/build.sh @@ -24,31 +24,18 @@ then mkdir /output/binary ||: # if exists mv /build/obj-*/programs/clickhouse* /output/binary - # Copy all fuzzers if any - FUZZER_TARGETS=$(find /build/obj-*/src -name '*_fuzzer' -execdir basename {} ';' | tr '\n' ' ') - - if [[ $FUZZER_TARGETS != "" ]]; then - - mkdir -p /output/fuzzers ||: # if exists - for FUZZER_TARGET in $FUZZER_TARGETS - do - FUZZER_PATH=$(find /build/obj-*/src -name "$FUZZER_TARGET") - strip --strip-unneeded "$FUZZER_PATH" - mv "$FUZZER_PATH" /output/fuzzers ||: # if exists - done - - - tar -zcvf /output/fuzzers.tar.gz /output/fuzzers - rm -rf /output/fuzzers - - fi - if [ "$BINARY_OUTPUT" = "tests" ] then mv /build/obj-*/src/unit_tests_dbms /output/binary fi fi +# Also build fuzzers if any sanitizer specified +if [ -n "$SANITIZER" ] +then + build/docker/packager/other/fuzzer.sh +fi + ccache --show-config ||: ccache --show-stats ||: diff --git a/docker/packager/other/fuzzer.sh b/docker/packager/other/fuzzer.sh new file mode 100644 index 00000000000..d6609d3c632 --- /dev/null +++ b/docker/packager/other/fuzzer.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -x -e + +# This script is responsible for building all fuzzers, and copy them to output directory +# as an archive. + +# Delete previous cache, because we add a new flags -DENABLE_FUZZING=1 and -DFUZZER=libfuzzer +rm -f CMakeCache.txt +read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}" +# Hope, that the most part of files will be in cache, so we just link new executables +cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" \ + "-DSANITIZE=$SANITIZER" -DENABLE_FUZZING=1 -DFUZZER='libfuzzer' -DENABLE_TCMALLOC=0 -DENABLE_JEMALLOC=0 \ + -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" .. + +FUZZER_TARGETS=$(find ../src -name '*_fuzzer.cpp' -execdir basename {} .cpp ';' | tr '\n' ' ') + +mkdir -p /output/fuzzers +for FUZZER_TARGET in $FUZZER_TARGETS +do + # shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty. + ninja $NINJA_FLAGS $FUZZER_TARGET + # Find this binary in build directory and strip it + FUZZER_PATH=$(find ./src -name "$FUZZER_TARGET") + strip --strip-unneeded "$FUZZER_PATH" + mv "$FUZZER_PATH" /output/fuzzers +done + +tar -zcvf /output/fuzzers.tar.gz /output/fuzzers +rm -rf /output/fuzzers diff --git a/release b/release index de549595d43..6e6970d7b00 100755 --- a/release +++ b/release @@ -60,9 +60,6 @@ then elif [[ "$SANITIZER" == "thread" ]]; then VERSION_POSTFIX+="+tsan" elif [[ "$SANITIZER" == "memory" ]]; then VERSION_POSTFIX+="+msan" elif [[ "$SANITIZER" == "undefined" ]]; then VERSION_POSTFIX+="+ubsan" - elif [[ "$SANITIZER" == "libfuzzer" ]]; then - VERSION_POSTFIX+="+libfuzzer" - MALLOC_OPTS="-DENABLE_TCMALLOC=0 -DENABLE_JEMALLOC=0" else echo "Unknown value of SANITIZER variable: $SANITIZER" exit 3