From 46fa7dbb8080aa9a2189a1fe0adbe3110f89c11f Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Tue, 5 Sep 2023 12:59:32 +0000 Subject: [PATCH 01/12] add libFuzzer build on 'libFuzzer' label, build produces artifacts --- .github/workflows/pull_request.yml | 45 +++++++++++ CMakeLists.txt | 22 ++---- cmake/utils.cmake | 120 +++++++++++++++++++++++++++++ docker/packager/binary/build.sh | 8 +- 4 files changed, 173 insertions(+), 22 deletions(-) create mode 100644 cmake/utils.cmake diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ce135846dd5..e6a4d1bf92e 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -5059,6 +5059,51 @@ jobs: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# +#################################### libFuzzer build ######################################## +############################################################################################# + BuilderFuzzers: + if: contains(github.event.pull_request.labels.*.name, 'libFuzzer') + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=fuzzers + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +############################################################################################# ###################################### JEPSEN TESTS ######################################### ############################################################################################# Jepsen: diff --git a/CMakeLists.txt b/CMakeLists.txt index 65ff9dc5384..781a9efe64a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,7 @@ include (cmake/tools.cmake) include (cmake/ccache.cmake) include (cmake/clang_tidy.cmake) include (cmake/git.cmake) +include (cmake/utils.cmake) # Ignore export() since we don't use it, # but it gets broken with a global targets via link_libraries() @@ -562,22 +563,6 @@ add_subdirectory (programs) add_subdirectory (tests) add_subdirectory (utils) -# Function get_all_targets collects all targets recursively -function(get_all_targets var) - macro(get_all_targets_recursive targets dir) - get_property(subdirectories DIRECTORY ${dir} PROPERTY SUBDIRECTORIES) - foreach(subdir ${subdirectories}) - get_all_targets_recursive(${targets} ${subdir}) - endforeach() - get_property(current_targets DIRECTORY ${dir} PROPERTY BUILDSYSTEM_TARGETS) - list(APPEND ${targets} ${current_targets}) - endmacro() - - set(targets) - get_all_targets_recursive(targets ${CMAKE_CURRENT_SOURCE_DIR}) - set(${var} ${targets} PARENT_SCOPE) -endfunction() - if (FUZZER) # Bundle fuzzers target add_custom_target(fuzzers) @@ -592,11 +577,14 @@ if (FUZZER) # clickhouse fuzzer isn't working correctly # initial PR https://github.com/ClickHouse/ClickHouse/pull/27526 #if (target MATCHES ".+_fuzzer" OR target STREQUAL "clickhouse") - if (target MATCHES ".+_fuzzer") + if (target_type STREQUAL "EXECUTABLE" AND target MATCHES ".+_fuzzer") message(STATUS "${target} instrumented with fuzzer") target_link_libraries(${target} PUBLIC ch_contrib::fuzzer) # Add to fuzzers bundle add_dependencies(fuzzers ${target}) + get_target_filename(${target} target_bin_name) + get_target_property(target_bin_dir ${target} BINARY_DIR) + add_custom_command(TARGET fuzzers POST_BUILD COMMAND mv "${target_bin_dir}/${target_bin_name}" "${CMAKE_CURRENT_BINARY_DIR}/programs/" VERBATIM) endif() endif() endforeach() diff --git a/cmake/utils.cmake b/cmake/utils.cmake new file mode 100644 index 00000000000..a318408098a --- /dev/null +++ b/cmake/utils.cmake @@ -0,0 +1,120 @@ +# Useful stuff + +# Function get_all_targets collects all targets recursively +function(get_all_targets outvar) + macro(get_all_targets_recursive targets dir) + get_property(subdirectories DIRECTORY ${dir} PROPERTY SUBDIRECTORIES) + foreach(subdir ${subdirectories}) + get_all_targets_recursive(${targets} ${subdir}) + endforeach() + get_property(current_targets DIRECTORY ${dir} PROPERTY BUILDSYSTEM_TARGETS) + list(APPEND ${targets} ${current_targets}) + endmacro() + + set(targets) + get_all_targets_recursive(targets ${CMAKE_CURRENT_SOURCE_DIR}) + set(${outvar} ${targets} PARENT_SCOPE) +endfunction() + + +# Function get_target_filename calculates target's output file name +function(get_target_filename target outvar) + get_target_property(prop_type "${target}" TYPE) + get_target_property(prop_is_framework "${target}" FRAMEWORK) + get_target_property(prop_outname "${target}" OUTPUT_NAME) + get_target_property(prop_archive_outname "${target}" ARCHIVE_OUTPUT_NAME) + get_target_property(prop_library_outname "${target}" LIBRARY_OUTPUT_NAME) + get_target_property(prop_runtime_outname "${target}" RUNTIME_OUTPUT_NAME) + # message("prop_archive_outname: ${prop_archive_outname}") + # message("prop_library_outname: ${prop_library_outname}") + # message("prop_runtime_outname: ${prop_runtime_outname}") + if(DEFINED CMAKE_BUILD_TYPE) + get_target_property(prop_cfg_outname "${target}" "${OUTPUT_NAME}_${CMAKE_BUILD_TYPE}") + get_target_property(prop_archive_cfg_outname "${target}" "${ARCHIVE_OUTPUT_NAME}_${CMAKE_BUILD_TYPE}") + get_target_property(prop_library_cfg_outname "${target}" "${LIBRARY_OUTPUT_NAME}_${CMAKE_BUILD_TYPE}") + get_target_property(prop_runtime_cfg_outname "${target}" "${RUNTIME_OUTPUT_NAME}_${CMAKE_BUILD_TYPE}") + # message("prop_archive_cfg_outname: ${prop_archive_cfg_outname}") + # message("prop_library_cfg_outname: ${prop_library_cfg_outname}") + # message("prop_runtime_cfg_outname: ${prop_runtime_cfg_outname}") + if(NOT ("${prop_cfg_outname}" STREQUAL "prop_cfg_outname-NOTFOUND")) + set(prop_outname "${prop_cfg_outname}") + endif() + if(NOT ("${prop_archive_cfg_outname}" STREQUAL "prop_archive_cfg_outname-NOTFOUND")) + set(prop_archive_outname "${prop_archive_cfg_outname}") + endif() + if(NOT ("${prop_library_cfg_outname}" STREQUAL "prop_library_cfg_outname-NOTFOUND")) + set(prop_library_outname "${prop_library_cfg_outname}") + endif() + if(NOT ("${prop_runtime_cfg_outname}" STREQUAL "prop_runtime_cfg_outname-NOTFOUND")) + set(prop_runtime_outname "${prop_runtime_cfg_outname}") + endif() + endif() + set(outname "${target}") + if(NOT ("${prop_outname}" STREQUAL "prop_outname-NOTFOUND")) + set(outname "${prop_outname}") + endif() + if("${prop_is_framework}") + set(filename "${outname}") + elseif(prop_type STREQUAL "STATIC_LIBRARY") + if(NOT ("${prop_archive_outname}" STREQUAL "prop_archive_outname-NOTFOUND")) + set(outname "${prop_archive_outname}") + endif() + set(filename "${CMAKE_STATIC_LIBRARY_PREFIX}${outname}${CMAKE_STATIC_LIBRARY_SUFFIX}") + elseif(prop_type STREQUAL "MODULE_LIBRARY") + if(NOT ("${prop_library_outname}" STREQUAL "prop_library_outname-NOTFOUND")) + set(outname "${prop_library_outname}") + endif() + set(filename "${CMAKE_SHARED_MODULE_LIBRARY_PREFIX}${outname}${CMAKE_SHARED_MODULE_LIBRARY_SUFFIX}") + elseif(prop_type STREQUAL "SHARED_LIBRARY") + if(WIN32) + if(NOT ("${prop_runtime_outname}" STREQUAL "prop_runtime_outname-NOTFOUND")) + set(outname "${prop_runtime_outname}") + endif() + else() + if(NOT ("${prop_library_outname}" STREQUAL "prop_library_outname-NOTFOUND")) + set(outname "${prop_library_outname}") + endif() + endif() + set(filename "${CMAKE_SHARED_LIBRARY_PREFIX}${outname}${CMAKE_SHARED_LIBRARY_SUFFIX}") + elseif(prop_type STREQUAL "EXECUTABLE") + if(NOT ("${prop_runtime_outname}" STREQUAL "prop_runtime_outname-NOTFOUND")) + set(outname "${prop_runtime_outname}") + endif() + set(filename "${CMAKE_EXECUTABLE_PREFIX}${outname}${CMAKE_EXECUTABLE_SUFFIX}") + else() + message(FATAL_ERROR "target \"${target}\" is not of type STATIC_LIBRARY, MODULE_LIBRARY, SHARED_LIBRARY, or EXECUTABLE.") + endif() + set("${outvar}" "${filename}" PARENT_SCOPE) +endfunction() + + +# Function get_cmake_properties returns list of all propreties that cmake supports +function(get_cmake_properties outvar) + execute_process(COMMAND cmake --help-property-list OUTPUT_VARIABLE cmake_properties) + # Convert command output into a CMake list + string(REGEX REPLACE ";" "\\\\;" cmake_properties "${cmake_properties}") + string(REGEX REPLACE "\n" ";" cmake_properties "${cmake_properties}") + list(REMOVE_DUPLICATES cmake_properties) + set("${outvar}" "${cmake_properties}" PARENT_SCOPE) +endfunction() + +# Function get_target_property_list returns list of all propreties set for target +function(get_target_property_list target outvar) + get_cmake_properties(cmake_property_list) + foreach(property ${cmake_property_list}) + string(REPLACE "" "${CMAKE_BUILD_TYPE}" property ${property}) + + # https://stackoverflow.com/questions/32197663/how-can-i-remove-the-the-location-property-may-not-be-read-from-target-error-i + if(property STREQUAL "LOCATION" OR property MATCHES "^LOCATION_" OR property MATCHES "_LOCATION$") + continue() + endif() + + get_property(was_set TARGET ${target} PROPERTY ${property} SET) + if(was_set) + get_target_property(value ${target} ${property}) + string(REGEX REPLACE ";" "\\\\\\\\;" value "${value}") + list(APPEND outvar "${property} = ${value}") + endif() + endforeach() + set(${outvar} ${${outvar}} PARENT_SCOPE) +endfunction() diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 39d299e1794..75a18528e65 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -97,11 +97,9 @@ if [ -n "$MAKE_DEB" ]; then bash -x /build/packages/build fi -if [ "$BUILD_TARGET" != "fuzzers" ]; then - mv ./programs/clickhouse* /output - [ -x ./programs/self-extracting/clickhouse ] && mv ./programs/self-extracting/clickhouse /output - mv ./src/unit_tests_dbms /output ||: # may not exist for some binary builds -fi +mv ./programs/clickhouse* /output || mv ./programs/*_fuzzer /output +[ -x ./programs/self-extracting/clickhouse ] && mv ./programs/self-extracting/clickhouse /output +mv ./src/unit_tests_dbms /output ||: # may not exist for some binary builds prepare_combined_output () { local OUTPUT From b7a17bf8dda0b94db456d2883e507d503f400594 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Sun, 10 Sep 2023 17:07:49 +0000 Subject: [PATCH 02/12] add libFuzzer tests, initial integration --- .github/workflows/pull_request.yml | 37 ++ docker/test/libfuzzer/Dockerfile | 42 +++ docker/test/libfuzzer/parse_options.py | 61 ++++ docker/test/libfuzzer/run_libfuzzer.sh | 115 +++++++ tests/ci/build_download_helper.py | 6 + tests/ci/ci_config.py | 1 + tests/ci/libfuzzer_test_check.py | 458 +++++++++++++++++++++++++ 7 files changed, 720 insertions(+) create mode 100644 docker/test/libfuzzer/Dockerfile create mode 100644 docker/test/libfuzzer/parse_options.py create mode 100644 docker/test/libfuzzer/run_libfuzzer.sh create mode 100644 tests/ci/libfuzzer_test_check.py diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index e6a4d1bf92e..7e56254bac0 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -5103,6 +5103,43 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +############################################################################################## +################################ libFuzzer TESTS ############################################# +############################################################################################## + libFuzzerTest: + needs: [BuilderFuzzers] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/libfuzzer + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=libFuzzer tests + REPO_COPY=${{runner.temp}}/libfuzzer/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: libFuzzer test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 libfuzzer_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" ############################################################################################# ###################################### JEPSEN TESTS ######################################### ############################################################################################# diff --git a/docker/test/libfuzzer/Dockerfile b/docker/test/libfuzzer/Dockerfile new file mode 100644 index 00000000000..77815431314 --- /dev/null +++ b/docker/test/libfuzzer/Dockerfile @@ -0,0 +1,42 @@ +ARG FROM_TAG=latest +FROM clickhouse/test-base:$FROM_TAG + +# ARG for quick switch to a given ubuntu mirror +ARG apt_archive="http://archive.ubuntu.com" +RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list + +ENV LANG=C.UTF-8 +ENV TZ=Europe/Amsterdam +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \ + ca-certificates \ + libc6-dbg \ + moreutils \ + ncdu \ + p7zip-full \ + parallel \ + psmisc \ + python3 \ + python3-pip \ + rsync \ + tree \ + tzdata \ + vim \ + wget \ + && apt-get autoremove --yes \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install Jinja2 + +COPY * / + +SHELL ["/bin/bash", "-c"] +CMD set -o pipefail \ + && cd /workspace \ + && timeout -s 9 1h /run_libfuzzer.sh 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log + +# docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/libfuzzer + diff --git a/docker/test/libfuzzer/parse_options.py b/docker/test/libfuzzer/parse_options.py new file mode 100644 index 00000000000..5695e80a714 --- /dev/null +++ b/docker/test/libfuzzer/parse_options.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +"""Helper script for parsing custom fuzzing options.""" +import configparser +import sys + + +def parse_options(options_file_path, options_section): + """Parses the given file and returns options from the given section.""" + parser = configparser.ConfigParser() + parser.read(options_file_path) + + if not parser.has_section(options_section): + return None + + options = parser[options_section] + + if options_section == "libfuzzer": + options_string = " ".join( + "-%s=%s" % (key, value) for key, value in options.items() + ) + else: + # Sanitizer options. + options_string = ":".join( + "%s=%s" % (key, value) for key, value in options.items() + ) + + return options_string + + +def main(): + """Processes the arguments and prints the options in the correct format.""" + if len(sys.argv) < 3: + sys.stderr.write( + "Usage: %s \n" % sys.argv[0] + ) + return 1 + + options = parse_options(sys.argv[1], sys.argv[2]) + if options is not None: + print(options) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/test/libfuzzer/run_libfuzzer.sh b/docker/test/libfuzzer/run_libfuzzer.sh new file mode 100644 index 00000000000..49a59dafb90 --- /dev/null +++ b/docker/test/libfuzzer/run_libfuzzer.sh @@ -0,0 +1,115 @@ +#!/bin/bash -eu +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +# Fuzzer runner. Appends .options arguments and seed corpus to users args. +# Usage: $0 + +export PATH=$OUT:$PATH +cd $OUT + +DEBUGGER=${DEBUGGER:-} + +FUZZER=$1 +shift + +# This env var is set by CIFuzz. CIFuzz fills this directory with the corpus +# from ClusterFuzz. +CORPUS_DIR=${CORPUS_DIR:-} +if [ -z "$CORPUS_DIR" ] +then + CORPUS_DIR="/tmp/${FUZZER}_corpus" + rm -rf $CORPUS_DIR && mkdir -p $CORPUS_DIR +fi + +SANITIZER=${SANITIZER:-} +if [ -z $SANITIZER ]; then + # If $SANITIZER is not specified (e.g. calling from `reproduce` command), it + # is not important and can be set to any value. + SANITIZER="default" +fi + +if [[ "$RUN_FUZZER_MODE" = interactive ]]; then + FUZZER_OUT="$OUT/${FUZZER}_${FUZZING_ENGINE}_${SANITIZER}_out" +else + FUZZER_OUT="/tmp/${FUZZER}_${FUZZING_ENGINE}_${SANITIZER}_out" +fi + + +rm -rf $FUZZER_OUT && mkdir -p $FUZZER_OUT + +SEED_CORPUS="${FUZZER}_seed_corpus.zip" + +# TODO: Investigate why this code block is skipped +# by all default fuzzers in bad_build_check. +# They all set SKIP_SEED_CORPUS=1. +if [ -f $SEED_CORPUS ] && [ -z ${SKIP_SEED_CORPUS:-} ]; then + echo "Using seed corpus: $SEED_CORPUS" + unzip -o -d ${CORPUS_DIR}/ $SEED_CORPUS > /dev/null +fi + +OPTIONS_FILE="${FUZZER}.options" +CUSTOM_LIBFUZZER_OPTIONS="" + +if [ -f $OPTIONS_FILE ]; then + custom_asan_options=$(parse_options.py $OPTIONS_FILE asan) + if [ ! -z $custom_asan_options ]; then + export ASAN_OPTIONS="$ASAN_OPTIONS:$custom_asan_options" + fi + + custom_msan_options=$(parse_options.py $OPTIONS_FILE msan) + if [ ! -z $custom_msan_options ]; then + export MSAN_OPTIONS="$MSAN_OPTIONS:$custom_msan_options" + fi + + custom_ubsan_options=$(parse_options.py $OPTIONS_FILE ubsan) + if [ ! -z $custom_ubsan_options ]; then + export UBSAN_OPTIONS="$UBSAN_OPTIONS:$custom_ubsan_options" + fi + + CUSTOM_LIBFUZZER_OPTIONS=$(parse_options.py $OPTIONS_FILE libfuzzer) +fi + + + +CMD_LINE="$OUT/$FUZZER $FUZZER_ARGS $*" + +if [ -z ${SKIP_SEED_CORPUS:-} ]; then +CMD_LINE="$CMD_LINE $CORPUS_DIR" +fi + +if [[ ! -z ${CUSTOM_LIBFUZZER_OPTIONS} ]]; then +CMD_LINE="$CMD_LINE $CUSTOM_LIBFUZZER_OPTIONS" +fi + +if [[ ! "$CMD_LINE" =~ "-dict=" ]]; then +if [ -f "$FUZZER.dict" ]; then + CMD_LINE="$CMD_LINE -dict=$FUZZER.dict" +fi +fi + +CMD_LINE="$CMD_LINE < /dev/null" + +echo $CMD_LINE + +# Unset OUT so the fuzz target can't rely on it. +unset OUT + +if [ ! -z "$DEBUGGER" ]; then + CMD_LINE="$DEBUGGER $CMD_LINE" +fi + +bash -c "$CMD_LINE" diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index a6fda749494..02e22e88a96 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -210,3 +210,9 @@ def download_performance_build(check_name, reports_path, result_path): result_path, lambda x: x.endswith("performance.tar.zst"), ) + + +def download_fuzzers(check_name, reports_path, result_path): + download_builds_filter( + check_name, reports_path, result_path, lambda x: x.endswith("_fuzzer") + ) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index b9ccc23cb2e..198395eca27 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -282,6 +282,7 @@ CI_CONFIG = CiConfig( "SQLancer (debug)": TestConfig("package_debug"), "Sqllogic test (release)": TestConfig("package_release"), "SQLTest": TestConfig("package_release"), + "libFuzzer tests": TestConfig("fuzzers"), }, ) CI_CONFIG.validate() diff --git a/tests/ci/libfuzzer_test_check.py b/tests/ci/libfuzzer_test_check.py new file mode 100644 index 00000000000..148b6e6d1e4 --- /dev/null +++ b/tests/ci/libfuzzer_test_check.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 + +import argparse +import csv +import logging +import os +import re +import subprocess +import sys +import atexit +from pathlib import Path +from typing import List, Tuple + +from github import Github + +# from build_download_helper import download_all_deb_packages +from build_download_helper import download_fuzzers +from clickhouse_helper import ( + CiLogsCredentials, + # ClickHouseHelper, + # prepare_tests_results_for_clickhouse, +) +from commit_status_helper import ( + # NotSet, + RerunHelper, + get_commit, + # override_status, + # post_commit_status, + # post_commit_status_to_file, + update_mergeable_check, +) +from docker_pull_helper import DockerImage # , get_image_with_version + +# from download_release_packages import download_last_release +from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH +from get_robot_token import get_best_robot_token +from pr_info import PRInfo # , FORCE_TESTS_LABEL +from report import TestResults, read_test_results + +# from s3_helper import S3Helper +from stopwatch import Stopwatch + +# from tee_popen import TeePopen +# from upload_result_helper import upload_results + +NO_CHANGES_MSG = "Nothing to run" + + +def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): + result = [] + if "DatabaseReplicated" in check_name: + result.append("USE_DATABASE_REPLICATED=1") + if "DatabaseOrdinary" in check_name: + result.append("USE_DATABASE_ORDINARY=1") + if "wide parts enabled" in check_name: + result.append("USE_POLYMORPHIC_PARTS=1") + if "ParallelReplicas" in check_name: + result.append("USE_PARALLEL_REPLICAS=1") + if "s3 storage" in check_name: + result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1") + if "analyzer" in check_name: + result.append("USE_NEW_ANALYZER=1") + + if run_by_hash_total != 0: + result.append(f"RUN_BY_HASH_NUM={run_by_hash_num}") + result.append(f"RUN_BY_HASH_TOTAL={run_by_hash_total}") + + return result + + +# def get_image_name(check_name): +# if "stateless" in check_name.lower(): +# return "clickhouse/stateless-test" +# if "stateful" in check_name.lower(): +# return "clickhouse/stateful-test" +# else: +# raise Exception(f"Cannot deduce image name based on check name {check_name}") + + +def get_run_command( + # check_name: str, + fuzzers_path: str, + repo_path: str, + result_path: str, + # server_log_path: str, + kill_timeout: int, + additional_envs: List[str], + ci_logs_args: str, + image: DockerImage, + # flaky_check: bool, + # tests_to_run: List[str], +) -> str: + additional_options = ["--hung-check"] + additional_options.append("--print-time") + + # if tests_to_run: + # additional_options += tests_to_run + + additional_options_str = ( + '-e ADDITIONAL_OPTIONS="' + " ".join(additional_options) + '"' + ) + + envs = [ + f"-e MAX_RUN_TIME={int(0.9 * kill_timeout)}", + # a static link, don't use S3_URL or S3_DOWNLOAD + '-e S3_URL="https://s3.amazonaws.com/clickhouse-datasets"', + ] + + # if flaky_check: + # envs.append("-e NUM_TRIES=100") + # envs.append("-e MAX_RUN_TIME=1800") + + envs += [f"-e {e}" for e in additional_envs] + + env_str = " ".join(envs) + # volume_with_broken_test = ( + # f"--volume={repo_path}/tests/analyzer_tech_debt.txt:/analyzer_tech_debt.txt " + # if "analyzer" in check_name + # else "" + # ) + + return ( + f"docker run --volume={fuzzers_path}:/fuzzers " + f"{ci_logs_args}" + f"--volume={repo_path}/tests:/usr/share/clickhouse-test " + # f"{volume_with_broken_test}" + f"--volume={result_path}:/test_output " + # f"--volume={server_log_path}:/var/log/clickhouse-server " + f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}" + ) + + +def get_tests_to_run(pr_info: PRInfo) -> List[str]: + result = set() + + if pr_info.changed_files is None: + return [] + + for fpath in pr_info.changed_files: + if re.match(r"tests/queries/0_stateless/[0-9]{5}", fpath): + logging.info("File '%s' is changed and seems like a test", fpath) + fname = fpath.split("/")[3] + fname_without_ext = os.path.splitext(fname)[0] + # add '.' to the end of the test name not to run all tests with the same prefix + # e.g. we changed '00001_some_name.reference' + # and we have ['00001_some_name.sh', '00001_some_name_2.sql'] + # so we want to run only '00001_some_name.sh' + result.add(fname_without_ext + ".") + elif "tests/queries/" in fpath: + # log suspicious changes from tests/ for debugging in case of any problems + logging.info("File '%s' is changed, but it doesn't look like a test", fpath) + return list(result) + + +def process_results( + result_folder: str, + server_log_path: str, +) -> Tuple[str, str, TestResults, List[str]]: + test_results = [] # type: TestResults + additional_files = [] + # Just upload all files from result_folder. + # If task provides processed results, then it's responsible for content of result_folder. + if os.path.exists(result_folder): + test_files = [ + f + for f in os.listdir(result_folder) + if os.path.isfile(os.path.join(result_folder, f)) + ] + additional_files = [os.path.join(result_folder, f) for f in test_files] + + if os.path.exists(server_log_path): + server_log_files = [ + f + for f in os.listdir(server_log_path) + if os.path.isfile(os.path.join(server_log_path, f)) + ] + additional_files = additional_files + [ + os.path.join(server_log_path, f) for f in server_log_files + ] + + status = [] + status_path = os.path.join(result_folder, "check_status.tsv") + if os.path.exists(status_path): + logging.info("Found test_results.tsv") + with open(status_path, "r", encoding="utf-8") as status_file: + status = list(csv.reader(status_file, delimiter="\t")) + + if len(status) != 1 or len(status[0]) != 2: + logging.info("Files in result folder %s", os.listdir(result_folder)) + return "error", "Invalid check_status.tsv", test_results, additional_files + state, description = status[0][0], status[0][1] + + try: + results_path = Path(result_folder) / "test_results.tsv" + + if results_path.exists(): + logging.info("Found test_results.tsv") + else: + logging.info("Files in result folder %s", os.listdir(result_folder)) + return "error", "Not found test_results.tsv", test_results, additional_files + + test_results = read_test_results(results_path) + if len(test_results) == 0: + return "error", "Empty test_results.tsv", test_results, additional_files + except Exception as e: + return ( + "error", + f"Cannot parse test_results.tsv ({e})", + test_results, + additional_files, + ) + + return state, description, test_results, additional_files + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("check_name") + parser.add_argument("kill_timeout", type=int) + parser.add_argument( + "--validate-bugfix", + action="store_true", + help="Check that added tests failed on latest stable", + ) + parser.add_argument( + "--post-commit-status", + default="commit_status", + choices=["commit_status", "file"], + help="Where to public post commit status", + ) + return parser.parse_args() + + +def docker_build_image(image_name: str, filepath: Path) -> DockerImage: + # context = filepath.parent + docker_image = DockerImage(image_name) + build_cmd = f"docker build --network=host -t {image_name} {filepath}" + logging.info("Will build image with cmd: '%s'", build_cmd) + subprocess.check_call( + build_cmd, + shell=True, + ) + return docker_image + + +def main(): + logging.basicConfig(level=logging.INFO) + + stopwatch = Stopwatch() + + temp_path = TEMP_PATH + repo_path = REPO_COPY + reports_path = REPORTS_PATH + # post_commit_path = os.path.join(temp_path, "functional_commit_status.tsv") + + args = parse_args() + check_name = args.check_name + kill_timeout = args.kill_timeout + validate_bugfix_check = args.validate_bugfix + + # flaky_check = "flaky" in check_name.lower() + + # run_changed_tests = flaky_check or validate_bugfix_check + run_changed_tests = validate_bugfix_check + gh = Github(get_best_robot_token(), per_page=100) + + # For validate_bugfix_check we need up to date information about labels, so pr_event_from_api is used + pr_info = PRInfo( + need_changed_files=run_changed_tests, pr_event_from_api=validate_bugfix_check + ) + + commit = get_commit(gh, pr_info.sha) + atexit.register(update_mergeable_check, gh, pr_info, check_name) + + if not os.path.exists(temp_path): + os.makedirs(temp_path) + + # if validate_bugfix_check and "pr-bugfix" not in pr_info.labels: + # if args.post_commit_status == "file": + # post_commit_status_to_file( + # post_commit_path, + # f"Skipped (no pr-bugfix in {pr_info.labels})", + # "success", + # "null", + # ) + # logging.info("Skipping '%s' (no pr-bugfix in %s)", check_name, pr_info.labels) + # sys.exit(0) + + if "RUN_BY_HASH_NUM" in os.environ: + run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM", "0")) + run_by_hash_total = int(os.getenv("RUN_BY_HASH_TOTAL", "0")) + check_name_with_group = ( + check_name + f" [{run_by_hash_num + 1}/{run_by_hash_total}]" + ) + else: + run_by_hash_num = 0 + run_by_hash_total = 0 + check_name_with_group = check_name + + rerun_helper = RerunHelper(commit, check_name_with_group) + if rerun_helper.is_already_finished_by_status(): + logging.info("Check is already finished according to github status, exiting") + sys.exit(0) + + # tests_to_run = [] + # if run_changed_tests: + # tests_to_run = get_tests_to_run(pr_info) + # if not tests_to_run: + # state = override_status("success", check_name, validate_bugfix_check) + # if args.post_commit_status == "commit_status": + # post_commit_status( + # commit, + # state, + # NotSet, + # NO_CHANGES_MSG, + # check_name_with_group, + # pr_info, + # ) + # elif args.post_commit_status == "file": + # post_commit_status_to_file( + # post_commit_path, + # description=NO_CHANGES_MSG, + # state=state, + # report_url="null", + # ) + # sys.exit(0) + + image_name = "clickhouse/libfuzzer-test" # get_image_name(check_name) + docker_image = docker_build_image( + image_name, Path("../../docker/test/libfuzzer/") + ) # get_image_with_version(reports_path, image_name) + + fuzzers_tmp_path = os.path.join(temp_path, "fuzzers_tmp") + if not os.path.exists(fuzzers_tmp_path): + os.makedirs(fuzzers_tmp_path) + + # if validate_bugfix_check: + # download_last_release(packages_path) + # else: + # download_all_deb_packages(check_name, reports_path, packages_path) + download_fuzzers(check_name, reports_path, fuzzers_tmp_path) + + fuzzers_path = os.path.join(temp_path, "fuzzers") + for fuzzer in os.listdir(fuzzers_tmp_path): + fuzzer_path = os.path.join(fuzzers_path, fuzzer) + os.makedirs(fuzzer_path) + os.rename( + os.path.join(fuzzers_tmp_path, fuzzer), os.path.join(fuzzer_path, fuzzer) + ) + + os.rmdir(fuzzers_tmp_path) + + # server_log_path = os.path.join(temp_path, "server_log") + # if not os.path.exists(server_log_path): + # os.makedirs(server_log_path) + + result_path = os.path.join(temp_path, "result_path") + if not os.path.exists(result_path): + os.makedirs(result_path) + + # run_log_path = os.path.join(result_path, "run.log") + + additional_envs = get_additional_envs( + check_name, run_by_hash_num, run_by_hash_total + ) + # if validate_bugfix_check: + # additional_envs.append("GLOBAL_TAGS=no-random-settings") + + ci_logs_credentials = CiLogsCredentials(Path(temp_path) / "export-logs-config.sh") + ci_logs_args = ci_logs_credentials.get_docker_arguments( + pr_info, stopwatch.start_time_str, check_name + ) + + run_command = get_run_command( + # check_name, + fuzzers_path, + repo_path, + result_path, + # server_log_path, + kill_timeout, + additional_envs, + ci_logs_args, + docker_image, + # flaky_check, + # tests_to_run, + ) + logging.info("Going to run func tests: %s", run_command) + + sys.exit(0) + + # with TeePopen(run_command, run_log_path) as process: + # retcode = process.wait() + # if retcode == 0: + # logging.info("Run successfully") + # else: + # logging.info("Run failed") + + # try: + # subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) + # except subprocess.CalledProcessError: + # logging.warning("Failed to change files owner in %s, ignoring it", temp_path) + + # ci_logs_credentials.clean_ci_logs_from_credentials(Path(run_log_path)) + # s3_helper = S3Helper() + + # state, description, test_results, additional_logs = process_results( + # result_path, server_log_path + # ) + # state = override_status(state, check_name, invert=validate_bugfix_check) + + # ch_helper = ClickHouseHelper() + + # report_url = upload_results( + # s3_helper, + # pr_info.number, + # pr_info.sha, + # test_results, + # [run_log_path] + additional_logs, + # check_name_with_group, + # ) + + # print(f"::notice:: {check_name} Report url: {report_url}") + # if args.post_commit_status == "commit_status": + # post_commit_status( + # commit, state, report_url, description, check_name_with_group, pr_info + # ) + # elif args.post_commit_status == "file": + # post_commit_status_to_file( + # post_commit_path, + # description, + # state, + # report_url, + # ) + # else: + # raise Exception( + # f'Unknown post_commit_status option "{args.post_commit_status}"' + # ) + + # prepared_events = prepare_tests_results_for_clickhouse( + # pr_info, + # test_results, + # state, + # stopwatch.duration_seconds, + # stopwatch.start_time_str, + # report_url, + # check_name_with_group, + # ) + # ch_helper.insert_events_into(db="default", table="checks", events=prepared_events) + + # if state != "success": + # if FORCE_TESTS_LABEL in pr_info.labels: + # print(f"'{FORCE_TESTS_LABEL}' enabled, will report success") + # else: + # sys.exit(1) + + +if __name__ == "__main__": + main() From 0847889db6e56f50d70e167915a69d4118b778b8 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Mon, 11 Sep 2023 19:06:00 +0000 Subject: [PATCH 03/12] libFuzzer infrastructure --- CMakeLists.txt | 1 + docker/packager/binary/build.sh | 1 + tests/ci/libfuzzer_test_check.py | 18 ++++-------------- tests/fuzz/build.sh | 28 ++++++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 14 deletions(-) create mode 100755 tests/fuzz/build.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 781a9efe64a..b4e13e8ab5b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -588,6 +588,7 @@ if (FUZZER) endif() endif() endforeach() + add_custom_command(TARGET fuzzers POST_BUILD COMMAND SRC=${CMAKE_SOURCE_DIR} BIN=${CMAKE_BINARY_DIR} OUT=${CMAKE_BINARY_DIR}/programs ${CMAKE_SOURCE_DIR}/tests/fuzz/build.sh VERBATIM) endif() include (cmake/sanitize_targets.cmake) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 75a18528e65..11efffd592c 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -100,6 +100,7 @@ fi mv ./programs/clickhouse* /output || mv ./programs/*_fuzzer /output [ -x ./programs/self-extracting/clickhouse ] && mv ./programs/self-extracting/clickhouse /output mv ./src/unit_tests_dbms /output ||: # may not exist for some binary builds +mv ./programs/*.dict ./programs/*.options ./programs/*_seed_corpus.zip /output ||: # libFuzzer oss-fuzz compatible infrastructure prepare_combined_output () { local OUTPUT diff --git a/tests/ci/libfuzzer_test_check.py b/tests/ci/libfuzzer_test_check.py index 148b6e6d1e4..75af6ddf5d9 100644 --- a/tests/ci/libfuzzer_test_check.py +++ b/tests/ci/libfuzzer_test_check.py @@ -330,25 +330,15 @@ def main(): image_name, Path("../../docker/test/libfuzzer/") ) # get_image_with_version(reports_path, image_name) - fuzzers_tmp_path = os.path.join(temp_path, "fuzzers_tmp") - if not os.path.exists(fuzzers_tmp_path): - os.makedirs(fuzzers_tmp_path) + fuzzers_path = os.path.join(temp_path, "fuzzers") + if not os.path.exists(fuzzers_path): + os.makedirs(fuzzers_path) # if validate_bugfix_check: # download_last_release(packages_path) # else: # download_all_deb_packages(check_name, reports_path, packages_path) - download_fuzzers(check_name, reports_path, fuzzers_tmp_path) - - fuzzers_path = os.path.join(temp_path, "fuzzers") - for fuzzer in os.listdir(fuzzers_tmp_path): - fuzzer_path = os.path.join(fuzzers_path, fuzzer) - os.makedirs(fuzzer_path) - os.rename( - os.path.join(fuzzers_tmp_path, fuzzer), os.path.join(fuzzer_path, fuzzer) - ) - - os.rmdir(fuzzers_tmp_path) + download_fuzzers(check_name, reports_path, fuzzers_path) # server_log_path = os.path.join(temp_path, "server_log") # if not os.path.exists(server_log_path): diff --git a/tests/fuzz/build.sh b/tests/fuzz/build.sh new file mode 100755 index 00000000000..12f41f6e079 --- /dev/null +++ b/tests/fuzz/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash -eu + +# copy fuzzer options and dictionaries +cp $SRC/tests/fuzz/*.dict $OUT/ +cp $SRC/tests/fuzz/*.options $OUT/ + +# prepare corpus dirs +mkdir -p $BIN/tests/fuzz/lexer_fuzzer.in/ +mkdir -p $BIN/tests/fuzz/select_parser_fuzzer.in/ +mkdir -p $BIN/tests/fuzz/create_parser_fuzzer.in/ +mkdir -p $BIN/tests/fuzz/execute_query_fuzzer.in/ + +# prepare corpus +cp $SRC/tests/queries/0_stateless/*.sql $BIN/tests/fuzz/lexer_fuzzer.in/ +cp $SRC/tests/queries/0_stateless/*.sql $BIN/tests/fuzz/select_parser_fuzzer.in/ +cp $SRC/tests/queries/0_stateless/*.sql $BIN/tests/fuzz/create_parser_fuzzer.in/ +cp $SRC/tests/queries/0_stateless/*.sql $BIN/tests/fuzz/execute_query_fuzzer.in/ +cp $SRC/tests/queries/1_stateful/*.sql $BIN/tests/fuzz/lexer_fuzzer.in/ +cp $SRC/tests/queries/1_stateful/*.sql $BIN/tests/fuzz/select_parser_fuzzer.in/ +cp $SRC/tests/queries/1_stateful/*.sql $BIN/tests/fuzz/create_parser_fuzzer.in/ +cp $SRC/tests/queries/1_stateful/*.sql $BIN/tests/fuzz/execute_query_fuzzer.in/ + +# build corpus archives +cd $BIN/tests/fuzz +for dir in *_fuzzer.in; do + fuzzer=$(basename $dir .in) + zip -rj "$OUT/${fuzzer}_seed_corpus.zip" "${dir}/" +done From eb3a7caa74748a5c9f97f3d740cefb16f9a4bae2 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Mon, 11 Sep 2023 21:10:03 +0000 Subject: [PATCH 04/12] add zip to build docker --- docker/packager/binary/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 940daad9c61..12818335807 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -74,6 +74,7 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \ python3-boto3 \ yasm \ zstd \ + zip \ && apt-get clean \ && rm -rf /var/lib/apt/lists From 44546458f09b913eb511eb1c332f06d0fad48a46 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Mon, 11 Sep 2023 22:45:50 +0000 Subject: [PATCH 05/12] add infrastructure files to the download filter --- tests/ci/build_download_helper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index 02e22e88a96..e27d10cbe5b 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -214,5 +214,8 @@ def download_performance_build(check_name, reports_path, result_path): def download_fuzzers(check_name, reports_path, result_path): download_builds_filter( - check_name, reports_path, result_path, lambda x: x.endswith("_fuzzer") + check_name, + reports_path, + result_path, + lambda x: x.endswith(("_fuzzer", ".dict", ".options", "_seed_corpus.zip")), ) From 1ad0a77c9f30146289bc640c1ba79de6db0d745e Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Tue, 12 Sep 2023 16:30:52 +0000 Subject: [PATCH 06/12] unzip corpora --- tests/ci/libfuzzer_test_check.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/ci/libfuzzer_test_check.py b/tests/ci/libfuzzer_test_check.py index 75af6ddf5d9..02f5c184b54 100644 --- a/tests/ci/libfuzzer_test_check.py +++ b/tests/ci/libfuzzer_test_check.py @@ -8,6 +8,7 @@ import re import subprocess import sys import atexit +import zipfile from pathlib import Path from typing import List, Tuple @@ -334,12 +335,15 @@ def main(): if not os.path.exists(fuzzers_path): os.makedirs(fuzzers_path) - # if validate_bugfix_check: - # download_last_release(packages_path) - # else: - # download_all_deb_packages(check_name, reports_path, packages_path) download_fuzzers(check_name, reports_path, fuzzers_path) + for file in os.listdir(fuzzers_path): + if file.endswith("_seed_corpus.zip"): + corpus_path = os.path.join( + temp_path, file.removesuffix("_seed_corpus.zip") + ".in" + ) + zipfile.ZipFile(os.path.join(temp_path, file), "r").extractall(corpus_path) + # server_log_path = os.path.join(temp_path, "server_log") # if not os.path.exists(server_log_path): # os.makedirs(server_log_path) @@ -374,7 +378,7 @@ def main(): # flaky_check, # tests_to_run, ) - logging.info("Going to run func tests: %s", run_command) + logging.info("Going to run libFuzzer tests: %s", run_command) sys.exit(0) From d80ae880606d2f40dae4dd9eb085a3016311a137 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Tue, 12 Sep 2023 20:56:42 +0000 Subject: [PATCH 07/12] run docker --- docker/test/libfuzzer/Dockerfile | 3 +- docker/test/libfuzzer/parse_options.py | 0 docker/test/libfuzzer/run_libfuzzer.sh | 142 +++++++++---------------- tests/ci/libfuzzer_test_check.py | 37 ++++--- 4 files changed, 77 insertions(+), 105 deletions(-) mode change 100644 => 100755 docker/test/libfuzzer/parse_options.py mode change 100644 => 100755 docker/test/libfuzzer/run_libfuzzer.sh diff --git a/docker/test/libfuzzer/Dockerfile b/docker/test/libfuzzer/Dockerfile index 77815431314..65cd8e4831f 100644 --- a/docker/test/libfuzzer/Dockerfile +++ b/docker/test/libfuzzer/Dockerfile @@ -33,9 +33,10 @@ RUN pip3 install Jinja2 COPY * / +ENV FUZZER_ARGS="-max_total_time=60" + SHELL ["/bin/bash", "-c"] CMD set -o pipefail \ - && cd /workspace \ && timeout -s 9 1h /run_libfuzzer.sh 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log # docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/libfuzzer diff --git a/docker/test/libfuzzer/parse_options.py b/docker/test/libfuzzer/parse_options.py old mode 100644 new mode 100755 diff --git a/docker/test/libfuzzer/run_libfuzzer.sh b/docker/test/libfuzzer/run_libfuzzer.sh old mode 100644 new mode 100755 index 49a59dafb90..b60e942f02a --- a/docker/test/libfuzzer/run_libfuzzer.sh +++ b/docker/test/libfuzzer/run_libfuzzer.sh @@ -1,115 +1,77 @@ #!/bin/bash -eu -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ # Fuzzer runner. Appends .options arguments and seed corpus to users args. # Usage: $0 -export PATH=$OUT:$PATH -cd $OUT +# export PATH=$OUT:$PATH +# cd $OUT DEBUGGER=${DEBUGGER:-} +FUZZER_ARGS=${FUZZER_ARGS:-} -FUZZER=$1 -shift +function run_fuzzer() { + FUZZER=$1 -# This env var is set by CIFuzz. CIFuzz fills this directory with the corpus -# from ClusterFuzz. -CORPUS_DIR=${CORPUS_DIR:-} -if [ -z "$CORPUS_DIR" ] -then - CORPUS_DIR="/tmp/${FUZZER}_corpus" - rm -rf $CORPUS_DIR && mkdir -p $CORPUS_DIR -fi + echo Running fuzzer "$FUZZER" -SANITIZER=${SANITIZER:-} -if [ -z $SANITIZER ]; then - # If $SANITIZER is not specified (e.g. calling from `reproduce` command), it - # is not important and can be set to any value. - SANITIZER="default" -fi + CORPUS_DIR="" + if [ -d "${FUZZER}.in" ]; then + CORPUS_DIR="${FUZZER}.in" + fi -if [[ "$RUN_FUZZER_MODE" = interactive ]]; then - FUZZER_OUT="$OUT/${FUZZER}_${FUZZING_ENGINE}_${SANITIZER}_out" -else - FUZZER_OUT="/tmp/${FUZZER}_${FUZZING_ENGINE}_${SANITIZER}_out" -fi + OPTIONS_FILE="${FUZZER}.options" + CUSTOM_LIBFUZZER_OPTIONS="" + if [ -f "$OPTIONS_FILE" ]; then + custom_asan_options=$(/parse_options.py "$OPTIONS_FILE" asan) + if [ -n "$custom_asan_options" ]; then + export ASAN_OPTIONS="$ASAN_OPTIONS:$custom_asan_options" + fi -rm -rf $FUZZER_OUT && mkdir -p $FUZZER_OUT + custom_msan_options=$(/parse_options.py "$OPTIONS_FILE" msan) + if [ -n "$custom_msan_options" ]; then + export MSAN_OPTIONS="$MSAN_OPTIONS:$custom_msan_options" + fi -SEED_CORPUS="${FUZZER}_seed_corpus.zip" + custom_ubsan_options=$(/parse_options.py "$OPTIONS_FILE" ubsan) + if [ -n "$custom_ubsan_options" ]; then + export UBSAN_OPTIONS="$UBSAN_OPTIONS:$custom_ubsan_options" + fi -# TODO: Investigate why this code block is skipped -# by all default fuzzers in bad_build_check. -# They all set SKIP_SEED_CORPUS=1. -if [ -f $SEED_CORPUS ] && [ -z ${SKIP_SEED_CORPUS:-} ]; then - echo "Using seed corpus: $SEED_CORPUS" - unzip -o -d ${CORPUS_DIR}/ $SEED_CORPUS > /dev/null -fi + CUSTOM_LIBFUZZER_OPTIONS=$(/parse_options.py "$OPTIONS_FILE" libfuzzer) + fi -OPTIONS_FILE="${FUZZER}.options" -CUSTOM_LIBFUZZER_OPTIONS="" + CMD_LINE="./$FUZZER $FUZZER_ARGS" + CMD_LINE="$CMD_LINE $CORPUS_DIR" -if [ -f $OPTIONS_FILE ]; then - custom_asan_options=$(parse_options.py $OPTIONS_FILE asan) - if [ ! -z $custom_asan_options ]; then - export ASAN_OPTIONS="$ASAN_OPTIONS:$custom_asan_options" - fi + if [[ -n "$CUSTOM_LIBFUZZER_OPTIONS" ]]; then + CMD_LINE="$CMD_LINE $CUSTOM_LIBFUZZER_OPTIONS" + fi - custom_msan_options=$(parse_options.py $OPTIONS_FILE msan) - if [ ! -z $custom_msan_options ]; then - export MSAN_OPTIONS="$MSAN_OPTIONS:$custom_msan_options" - fi + if [[ ! "$CMD_LINE" =~ "-dict=" ]]; then + if [ -f "$FUZZER.dict" ]; then + CMD_LINE="$CMD_LINE -dict=$FUZZER.dict" + fi + fi - custom_ubsan_options=$(parse_options.py $OPTIONS_FILE ubsan) - if [ ! -z $custom_ubsan_options ]; then - export UBSAN_OPTIONS="$UBSAN_OPTIONS:$custom_ubsan_options" - fi + CMD_LINE="$CMD_LINE < /dev/null" - CUSTOM_LIBFUZZER_OPTIONS=$(parse_options.py $OPTIONS_FILE libfuzzer) -fi + echo "$CMD_LINE" + # Unset OUT so the fuzz target can't rely on it. + # unset OUT + if [ -n "$DEBUGGER" ]; then + CMD_LINE="$DEBUGGER $CMD_LINE" + fi -CMD_LINE="$OUT/$FUZZER $FUZZER_ARGS $*" + bash -c "$CMD_LINE" +} -if [ -z ${SKIP_SEED_CORPUS:-} ]; then -CMD_LINE="$CMD_LINE $CORPUS_DIR" -fi +ls -al -if [[ ! -z ${CUSTOM_LIBFUZZER_OPTIONS} ]]; then -CMD_LINE="$CMD_LINE $CUSTOM_LIBFUZZER_OPTIONS" -fi - -if [[ ! "$CMD_LINE" =~ "-dict=" ]]; then -if [ -f "$FUZZER.dict" ]; then - CMD_LINE="$CMD_LINE -dict=$FUZZER.dict" -fi -fi - -CMD_LINE="$CMD_LINE < /dev/null" - -echo $CMD_LINE - -# Unset OUT so the fuzz target can't rely on it. -unset OUT - -if [ ! -z "$DEBUGGER" ]; then - CMD_LINE="$DEBUGGER $CMD_LINE" -fi - -bash -c "$CMD_LINE" +for fuzzer in *_fuzzer; do + if [ -f "$fuzzer" ] && [ -x "$fuzzer" ]; then + run_fuzzer "$fuzzer" + fi +done diff --git a/tests/ci/libfuzzer_test_check.py b/tests/ci/libfuzzer_test_check.py index 02f5c184b54..e7f907d02d4 100644 --- a/tests/ci/libfuzzer_test_check.py +++ b/tests/ci/libfuzzer_test_check.py @@ -41,7 +41,8 @@ from report import TestResults, read_test_results # from s3_helper import S3Helper from stopwatch import Stopwatch -# from tee_popen import TeePopen +from tee_popen import TeePopen + # from upload_result_helper import upload_results NO_CHANGES_MSG = "Nothing to run" @@ -121,8 +122,10 @@ def get_run_command( # ) return ( - f"docker run --volume={fuzzers_path}:/fuzzers " - f"{ci_logs_args}" + f"docker run " + f"{ci_logs_args} " + f"--workdir=/fuzzers " + f"--volume={fuzzers_path}:/fuzzers " f"--volume={repo_path}/tests:/usr/share/clickhouse-test " # f"{volume_with_broken_test}" f"--volume={result_path}:/test_output " @@ -338,11 +341,15 @@ def main(): download_fuzzers(check_name, reports_path, fuzzers_path) for file in os.listdir(fuzzers_path): - if file.endswith("_seed_corpus.zip"): + if file.endswith("_fuzzer"): + os.chmod(os.path.join(fuzzers_path, file), 0o777) + elif file.endswith("_seed_corpus.zip"): corpus_path = os.path.join( - temp_path, file.removesuffix("_seed_corpus.zip") + ".in" + fuzzers_path, file.removesuffix("_seed_corpus.zip") + ".in" + ) + zipfile.ZipFile(os.path.join(fuzzers_path, file), "r").extractall( + corpus_path ) - zipfile.ZipFile(os.path.join(temp_path, file), "r").extractall(corpus_path) # server_log_path = os.path.join(temp_path, "server_log") # if not os.path.exists(server_log_path): @@ -352,7 +359,7 @@ def main(): if not os.path.exists(result_path): os.makedirs(result_path) - # run_log_path = os.path.join(result_path, "run.log") + run_log_path = os.path.join(result_path, "run.log") additional_envs = get_additional_envs( check_name, run_by_hash_num, run_by_hash_total @@ -380,14 +387,16 @@ def main(): ) logging.info("Going to run libFuzzer tests: %s", run_command) - sys.exit(0) + # sys.exit(0) - # with TeePopen(run_command, run_log_path) as process: - # retcode = process.wait() - # if retcode == 0: - # logging.info("Run successfully") - # else: - # logging.info("Run failed") + with TeePopen(run_command, run_log_path) as process: + retcode = process.wait() + if retcode == 0: + logging.info("Run successfully") + else: + logging.info("Run failed") + + sys.exit(0) # try: # subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) From cd0c775355e2eb2b620a638a1d2ce3c6f83c7f1c Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Thu, 14 Sep 2023 03:10:55 +0000 Subject: [PATCH 08/12] review suggestions --- docker/images.json | 4 ++ docker/test/libfuzzer/Dockerfile | 2 +- docker/test/libfuzzer/run_libfuzzer.py | 73 ++++++++++++++++++++++++++ tests/ci/libfuzzer_test_check.py | 11 ++-- 4 files changed, 84 insertions(+), 6 deletions(-) create mode 100755 docker/test/libfuzzer/run_libfuzzer.py diff --git a/docker/images.json b/docker/images.json index d895e2da2f0..bddfd49ea3b 100644 --- a/docker/images.json +++ b/docker/images.json @@ -21,6 +21,10 @@ "name": "clickhouse/fuzzer", "dependent": [] }, + "docker/test/libfuzzer": { + "name": "clickhouse/libfuzzer", + "dependent": [] + }, "docker/test/performance-comparison": { "name": "clickhouse/performance-comparison", "dependent": [] diff --git a/docker/test/libfuzzer/Dockerfile b/docker/test/libfuzzer/Dockerfile index 65cd8e4831f..081cf5473f8 100644 --- a/docker/test/libfuzzer/Dockerfile +++ b/docker/test/libfuzzer/Dockerfile @@ -37,7 +37,7 @@ ENV FUZZER_ARGS="-max_total_time=60" SHELL ["/bin/bash", "-c"] CMD set -o pipefail \ - && timeout -s 9 1h /run_libfuzzer.sh 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log + && timeout -s 9 1h /run_libfuzzer.py 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log # docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/libfuzzer diff --git a/docker/test/libfuzzer/run_libfuzzer.py b/docker/test/libfuzzer/run_libfuzzer.py new file mode 100755 index 00000000000..b608c97de60 --- /dev/null +++ b/docker/test/libfuzzer/run_libfuzzer.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 + +import logging +import os +from pathlib import Path +import subprocess +from parse_options import parse_options + +DEBUGGER = os.getenv("DEBUGGER", "") +FUZZER_ARGS = os.getenv("FUZZER_ARGS", "") + + +def run_fuzzer(fuzzer: str): + logging.info(f"Running fuzzer {fuzzer}...") + + corpus_dir = f"{fuzzer}.in" + with Path(corpus_dir) as path: + if not path.exists() or not path.is_dir(): + corpus_dir = "" + + options_file = f"{fuzzer}.options" + custom_libfuzzer_options = "" + + with Path(options_file) as path: + if path.exists() and path.is_file(): + custom_asan_options = parse_options(options_file, "asan") + if custom_asan_options: + os.environ[ + "ASAN_OPTIONS" + ] = f"{os.environ['ASAN_OPTIONS']}:{custom_asan_options}" + + custom_msan_options = parse_options(options_file, "msan") + if custom_msan_options: + os.environ[ + "MSAN_OPTIONS" + ] = f"{os.environ['MSAN_OPTIONS']}:{custom_msan_options}" + + custom_ubsan_options = parse_options(options_file, "ubsan") + if custom_ubsan_options: + os.environ[ + "UBSAN_OPTIONS" + ] = f"{os.environ['UBSAN_OPTIONS']}:{custom_ubsan_options}" + + custom_libfuzzer_options = parse_options(options_file, "libfuzzer") + + cmd_line = f"{DEBUGGER} ./{fuzzer} {FUZZER_ARGS} {corpus_dir}" + if custom_libfuzzer_options: + cmd_line += f" {custom_libfuzzer_options}" + + if not "-dict=" in cmd_line and Path(f"{fuzzer}.dict").exists(): + cmd_line += f" -dict={fuzzer}.dict" + + cmd_line += " < /dev/null" + + logging.info(f"...will execute: {cmd_line}") + subprocess.check_call(cmd_line, shell=True) + + +def main(): + logging.basicConfig(level=logging.INFO) + + subprocess.check_call("ls -al", shell=True) + + with Path() as current: + for fuzzer in current.iterdir(): + if (current / fuzzer).is_file() and os.access(current / fuzzer, os.X_OK): + run_fuzzer(fuzzer) + + exit(0) + + +if __name__ == "__main__": + main() diff --git a/tests/ci/libfuzzer_test_check.py b/tests/ci/libfuzzer_test_check.py index e7f907d02d4..41d08dade77 100644 --- a/tests/ci/libfuzzer_test_check.py +++ b/tests/ci/libfuzzer_test_check.py @@ -30,7 +30,7 @@ from commit_status_helper import ( # post_commit_status_to_file, update_mergeable_check, ) -from docker_pull_helper import DockerImage # , get_image_with_version +from docker_pull_helper import DockerImage, get_image_with_version # from download_release_packages import download_last_release from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH @@ -329,10 +329,11 @@ def main(): # ) # sys.exit(0) - image_name = "clickhouse/libfuzzer-test" # get_image_name(check_name) - docker_image = docker_build_image( - image_name, Path("../../docker/test/libfuzzer/") - ) # get_image_with_version(reports_path, image_name) + # image_name = "clickhouse/libfuzzer-test" # get_image_name(check_name) + # docker_image = docker_build_image( + # image_name, Path("../../docker/test/libfuzzer/") + # ) + docker_image = get_image_with_version(reports_path, "clickhouse/libfuzzer") fuzzers_path = os.path.join(temp_path, "fuzzers") if not os.path.exists(fuzzers_path): From 3a14bde95a54759cf8af6f1aac6d730dc2f3aad3 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Thu, 14 Sep 2023 20:06:53 +0000 Subject: [PATCH 09/12] cleanup, fix tee to escape non-decodable symbols --- docker/test/libfuzzer/run_libfuzzer.sh | 77 ------- tests/ci/libfuzzer_test_check.py | 270 ++----------------------- tests/ci/tee_popen.py | 1 + 3 files changed, 17 insertions(+), 331 deletions(-) delete mode 100755 docker/test/libfuzzer/run_libfuzzer.sh diff --git a/docker/test/libfuzzer/run_libfuzzer.sh b/docker/test/libfuzzer/run_libfuzzer.sh deleted file mode 100755 index b60e942f02a..00000000000 --- a/docker/test/libfuzzer/run_libfuzzer.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -eu - -# Fuzzer runner. Appends .options arguments and seed corpus to users args. -# Usage: $0 - -# export PATH=$OUT:$PATH -# cd $OUT - -DEBUGGER=${DEBUGGER:-} -FUZZER_ARGS=${FUZZER_ARGS:-} - -function run_fuzzer() { - FUZZER=$1 - - echo Running fuzzer "$FUZZER" - - CORPUS_DIR="" - if [ -d "${FUZZER}.in" ]; then - CORPUS_DIR="${FUZZER}.in" - fi - - OPTIONS_FILE="${FUZZER}.options" - CUSTOM_LIBFUZZER_OPTIONS="" - - if [ -f "$OPTIONS_FILE" ]; then - custom_asan_options=$(/parse_options.py "$OPTIONS_FILE" asan) - if [ -n "$custom_asan_options" ]; then - export ASAN_OPTIONS="$ASAN_OPTIONS:$custom_asan_options" - fi - - custom_msan_options=$(/parse_options.py "$OPTIONS_FILE" msan) - if [ -n "$custom_msan_options" ]; then - export MSAN_OPTIONS="$MSAN_OPTIONS:$custom_msan_options" - fi - - custom_ubsan_options=$(/parse_options.py "$OPTIONS_FILE" ubsan) - if [ -n "$custom_ubsan_options" ]; then - export UBSAN_OPTIONS="$UBSAN_OPTIONS:$custom_ubsan_options" - fi - - CUSTOM_LIBFUZZER_OPTIONS=$(/parse_options.py "$OPTIONS_FILE" libfuzzer) - fi - - CMD_LINE="./$FUZZER $FUZZER_ARGS" - CMD_LINE="$CMD_LINE $CORPUS_DIR" - - if [[ -n "$CUSTOM_LIBFUZZER_OPTIONS" ]]; then - CMD_LINE="$CMD_LINE $CUSTOM_LIBFUZZER_OPTIONS" - fi - - if [[ ! "$CMD_LINE" =~ "-dict=" ]]; then - if [ -f "$FUZZER.dict" ]; then - CMD_LINE="$CMD_LINE -dict=$FUZZER.dict" - fi - fi - - CMD_LINE="$CMD_LINE < /dev/null" - - echo "$CMD_LINE" - - # Unset OUT so the fuzz target can't rely on it. - # unset OUT - - if [ -n "$DEBUGGER" ]; then - CMD_LINE="$DEBUGGER $CMD_LINE" - fi - - bash -c "$CMD_LINE" -} - -ls -al - -for fuzzer in *_fuzzer; do - if [ -f "$fuzzer" ] && [ -x "$fuzzer" ]; then - run_fuzzer "$fuzzer" - fi -done diff --git a/tests/ci/libfuzzer_test_check.py b/tests/ci/libfuzzer_test_check.py index 41d08dade77..9fee997cc96 100644 --- a/tests/ci/libfuzzer_test_check.py +++ b/tests/ci/libfuzzer_test_check.py @@ -1,49 +1,37 @@ #!/usr/bin/env python3 import argparse -import csv import logging import os -import re import subprocess import sys import atexit import zipfile from pathlib import Path -from typing import List, Tuple +from typing import List from github import Github -# from build_download_helper import download_all_deb_packages from build_download_helper import download_fuzzers from clickhouse_helper import ( CiLogsCredentials, - # ClickHouseHelper, - # prepare_tests_results_for_clickhouse, ) from commit_status_helper import ( - # NotSet, RerunHelper, get_commit, - # override_status, - # post_commit_status, - # post_commit_status_to_file, update_mergeable_check, ) from docker_pull_helper import DockerImage, get_image_with_version -# from download_release_packages import download_last_release from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH from get_robot_token import get_best_robot_token -from pr_info import PRInfo # , FORCE_TESTS_LABEL -from report import TestResults, read_test_results +from pr_info import PRInfo +from report import TestResults -# from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen -# from upload_result_helper import upload_results NO_CHANGES_MSG = "Nothing to run" @@ -70,34 +58,18 @@ def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): return result -# def get_image_name(check_name): -# if "stateless" in check_name.lower(): -# return "clickhouse/stateless-test" -# if "stateful" in check_name.lower(): -# return "clickhouse/stateful-test" -# else: -# raise Exception(f"Cannot deduce image name based on check name {check_name}") - - def get_run_command( - # check_name: str, - fuzzers_path: str, - repo_path: str, - result_path: str, - # server_log_path: str, + fuzzers_path: Path, + repo_path: Path, + result_path: Path, kill_timeout: int, additional_envs: List[str], ci_logs_args: str, image: DockerImage, - # flaky_check: bool, - # tests_to_run: List[str], ) -> str: additional_options = ["--hung-check"] additional_options.append("--print-time") - # if tests_to_run: - # additional_options += tests_to_run - additional_options_str = ( '-e ADDITIONAL_OPTIONS="' + " ".join(additional_options) + '"' ) @@ -108,18 +80,9 @@ def get_run_command( '-e S3_URL="https://s3.amazonaws.com/clickhouse-datasets"', ] - # if flaky_check: - # envs.append("-e NUM_TRIES=100") - # envs.append("-e MAX_RUN_TIME=1800") - envs += [f"-e {e}" for e in additional_envs] env_str = " ".join(envs) - # volume_with_broken_test = ( - # f"--volume={repo_path}/tests/analyzer_tech_debt.txt:/analyzer_tech_debt.txt " - # if "analyzer" in check_name - # else "" - # ) return ( f"docker run " @@ -127,96 +90,11 @@ def get_run_command( f"--workdir=/fuzzers " f"--volume={fuzzers_path}:/fuzzers " f"--volume={repo_path}/tests:/usr/share/clickhouse-test " - # f"{volume_with_broken_test}" f"--volume={result_path}:/test_output " - # f"--volume={server_log_path}:/var/log/clickhouse-server " f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}" ) -def get_tests_to_run(pr_info: PRInfo) -> List[str]: - result = set() - - if pr_info.changed_files is None: - return [] - - for fpath in pr_info.changed_files: - if re.match(r"tests/queries/0_stateless/[0-9]{5}", fpath): - logging.info("File '%s' is changed and seems like a test", fpath) - fname = fpath.split("/")[3] - fname_without_ext = os.path.splitext(fname)[0] - # add '.' to the end of the test name not to run all tests with the same prefix - # e.g. we changed '00001_some_name.reference' - # and we have ['00001_some_name.sh', '00001_some_name_2.sql'] - # so we want to run only '00001_some_name.sh' - result.add(fname_without_ext + ".") - elif "tests/queries/" in fpath: - # log suspicious changes from tests/ for debugging in case of any problems - logging.info("File '%s' is changed, but it doesn't look like a test", fpath) - return list(result) - - -def process_results( - result_folder: str, - server_log_path: str, -) -> Tuple[str, str, TestResults, List[str]]: - test_results = [] # type: TestResults - additional_files = [] - # Just upload all files from result_folder. - # If task provides processed results, then it's responsible for content of result_folder. - if os.path.exists(result_folder): - test_files = [ - f - for f in os.listdir(result_folder) - if os.path.isfile(os.path.join(result_folder, f)) - ] - additional_files = [os.path.join(result_folder, f) for f in test_files] - - if os.path.exists(server_log_path): - server_log_files = [ - f - for f in os.listdir(server_log_path) - if os.path.isfile(os.path.join(server_log_path, f)) - ] - additional_files = additional_files + [ - os.path.join(server_log_path, f) for f in server_log_files - ] - - status = [] - status_path = os.path.join(result_folder, "check_status.tsv") - if os.path.exists(status_path): - logging.info("Found test_results.tsv") - with open(status_path, "r", encoding="utf-8") as status_file: - status = list(csv.reader(status_file, delimiter="\t")) - - if len(status) != 1 or len(status[0]) != 2: - logging.info("Files in result folder %s", os.listdir(result_folder)) - return "error", "Invalid check_status.tsv", test_results, additional_files - state, description = status[0][0], status[0][1] - - try: - results_path = Path(result_folder) / "test_results.tsv" - - if results_path.exists(): - logging.info("Found test_results.tsv") - else: - logging.info("Files in result folder %s", os.listdir(result_folder)) - return "error", "Not found test_results.tsv", test_results, additional_files - - test_results = read_test_results(results_path) - if len(test_results) == 0: - return "error", "Empty test_results.tsv", test_results, additional_files - except Exception as e: - return ( - "error", - f"Cannot parse test_results.tsv ({e})", - test_results, - additional_files, - ) - - return state, description, test_results, additional_files - - def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("check_name") @@ -255,16 +133,12 @@ def main(): temp_path = TEMP_PATH repo_path = REPO_COPY reports_path = REPORTS_PATH - # post_commit_path = os.path.join(temp_path, "functional_commit_status.tsv") args = parse_args() check_name = args.check_name kill_timeout = args.kill_timeout validate_bugfix_check = args.validate_bugfix - # flaky_check = "flaky" in check_name.lower() - - # run_changed_tests = flaky_check or validate_bugfix_check run_changed_tests = validate_bugfix_check gh = Github(get_best_robot_token(), per_page=100) @@ -276,20 +150,9 @@ def main(): commit = get_commit(gh, pr_info.sha) atexit.register(update_mergeable_check, gh, pr_info, check_name) - if not os.path.exists(temp_path): + if not Path(temp_path).exists(): os.makedirs(temp_path) - # if validate_bugfix_check and "pr-bugfix" not in pr_info.labels: - # if args.post_commit_status == "file": - # post_commit_status_to_file( - # post_commit_path, - # f"Skipped (no pr-bugfix in {pr_info.labels})", - # "success", - # "null", - # ) - # logging.info("Skipping '%s' (no pr-bugfix in %s)", check_name, pr_info.labels) - # sys.exit(0) - if "RUN_BY_HASH_NUM" in os.environ: run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM", "0")) run_by_hash_total = int(os.getenv("RUN_BY_HASH_TOTAL", "0")) @@ -306,67 +169,30 @@ def main(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) - # tests_to_run = [] - # if run_changed_tests: - # tests_to_run = get_tests_to_run(pr_info) - # if not tests_to_run: - # state = override_status("success", check_name, validate_bugfix_check) - # if args.post_commit_status == "commit_status": - # post_commit_status( - # commit, - # state, - # NotSet, - # NO_CHANGES_MSG, - # check_name_with_group, - # pr_info, - # ) - # elif args.post_commit_status == "file": - # post_commit_status_to_file( - # post_commit_path, - # description=NO_CHANGES_MSG, - # state=state, - # report_url="null", - # ) - # sys.exit(0) - - # image_name = "clickhouse/libfuzzer-test" # get_image_name(check_name) - # docker_image = docker_build_image( - # image_name, Path("../../docker/test/libfuzzer/") - # ) docker_image = get_image_with_version(reports_path, "clickhouse/libfuzzer") - fuzzers_path = os.path.join(temp_path, "fuzzers") - if not os.path.exists(fuzzers_path): + fuzzers_path = Path(temp_path) / "fuzzers" + if not fuzzers_path.exists(): os.makedirs(fuzzers_path) download_fuzzers(check_name, reports_path, fuzzers_path) for file in os.listdir(fuzzers_path): if file.endswith("_fuzzer"): - os.chmod(os.path.join(fuzzers_path, file), 0o777) + os.chmod(fuzzers_path / file, 0o777) elif file.endswith("_seed_corpus.zip"): - corpus_path = os.path.join( - fuzzers_path, file.removesuffix("_seed_corpus.zip") + ".in" - ) - zipfile.ZipFile(os.path.join(fuzzers_path, file), "r").extractall( - corpus_path - ) + corpus_path = fuzzers_path / (file.removesuffix("_seed_corpus.zip") + ".in") + zipfile.ZipFile(fuzzers_path / file, "r").extractall(corpus_path) - # server_log_path = os.path.join(temp_path, "server_log") - # if not os.path.exists(server_log_path): - # os.makedirs(server_log_path) - - result_path = os.path.join(temp_path, "result_path") - if not os.path.exists(result_path): + result_path = Path(temp_path) / "result_path" + if not result_path.exists(): os.makedirs(result_path) - run_log_path = os.path.join(result_path, "run.log") + run_log_path = result_path / "run.log" additional_envs = get_additional_envs( check_name, run_by_hash_num, run_by_hash_total ) - # if validate_bugfix_check: - # additional_envs.append("GLOBAL_TAGS=no-random-settings") ci_logs_credentials = CiLogsCredentials(Path(temp_path) / "export-logs-config.sh") ci_logs_args = ci_logs_credentials.get_docker_arguments( @@ -374,22 +200,16 @@ def main(): ) run_command = get_run_command( - # check_name, fuzzers_path, - repo_path, + Path(repo_path), result_path, - # server_log_path, kill_timeout, additional_envs, ci_logs_args, docker_image, - # flaky_check, - # tests_to_run, ) logging.info("Going to run libFuzzer tests: %s", run_command) - # sys.exit(0) - with TeePopen(run_command, run_log_path) as process: retcode = process.wait() if retcode == 0: @@ -399,64 +219,6 @@ def main(): sys.exit(0) - # try: - # subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) - # except subprocess.CalledProcessError: - # logging.warning("Failed to change files owner in %s, ignoring it", temp_path) - - # ci_logs_credentials.clean_ci_logs_from_credentials(Path(run_log_path)) - # s3_helper = S3Helper() - - # state, description, test_results, additional_logs = process_results( - # result_path, server_log_path - # ) - # state = override_status(state, check_name, invert=validate_bugfix_check) - - # ch_helper = ClickHouseHelper() - - # report_url = upload_results( - # s3_helper, - # pr_info.number, - # pr_info.sha, - # test_results, - # [run_log_path] + additional_logs, - # check_name_with_group, - # ) - - # print(f"::notice:: {check_name} Report url: {report_url}") - # if args.post_commit_status == "commit_status": - # post_commit_status( - # commit, state, report_url, description, check_name_with_group, pr_info - # ) - # elif args.post_commit_status == "file": - # post_commit_status_to_file( - # post_commit_path, - # description, - # state, - # report_url, - # ) - # else: - # raise Exception( - # f'Unknown post_commit_status option "{args.post_commit_status}"' - # ) - - # prepared_events = prepare_tests_results_for_clickhouse( - # pr_info, - # test_results, - # state, - # stopwatch.duration_seconds, - # stopwatch.start_time_str, - # report_url, - # check_name_with_group, - # ) - # ch_helper.insert_events_into(db="default", table="checks", events=prepared_events) - - # if state != "success": - # if FORCE_TESTS_LABEL in pr_info.labels: - # print(f"'{FORCE_TESTS_LABEL}' enabled, will report success") - # else: - # sys.exit(1) - if __name__ == "__main__": main() diff --git a/tests/ci/tee_popen.py b/tests/ci/tee_popen.py index 7872b489951..a50532aea20 100644 --- a/tests/ci/tee_popen.py +++ b/tests/ci/tee_popen.py @@ -55,6 +55,7 @@ class TeePopen: stderr=STDOUT, stdout=PIPE, bufsize=1, + errors="backslashreplace", ) if self.timeout is not None and self.timeout > 0: t = Thread(target=self._check_timeout) From d1cd3cdd2a4dbbd7d695c09bdb09b7b6d1830400 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Sat, 16 Sep 2023 01:04:19 +0000 Subject: [PATCH 10/12] move on its own workflow --- .github/workflows/libfuzzer.yml | 80 +++++++++++++++++++++++++++++ .github/workflows/pull_request.yml | 82 ------------------------------ 2 files changed, 80 insertions(+), 82 deletions(-) create mode 100644 .github/workflows/libfuzzer.yml diff --git a/.github/workflows/libfuzzer.yml b/.github/workflows/libfuzzer.yml new file mode 100644 index 00000000000..74772ccf6d9 --- /dev/null +++ b/.github/workflows/libfuzzer.yml @@ -0,0 +1,80 @@ +name: libFuzzer + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + +on: # yamllint disable-line rule:truthy +# schedule: +# - cron: '0 0 2 31 1' # never for now + workflow_dispatch: +jobs: + BuilderFuzzers: + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=fuzzers + EOF + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + libFuzzerTest: + needs: [BuilderFuzzers] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/libfuzzer + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=libFuzzer tests + REPO_COPY=${{runner.temp}}/libfuzzer/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: libFuzzer test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 libfuzzer_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7e56254bac0..ce135846dd5 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -5059,88 +5059,6 @@ jobs: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# -#################################### libFuzzer build ######################################## -############################################################################################# - BuilderFuzzers: - if: contains(github.event.pull_request.labels.*.name, 'libFuzzer') - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=fuzzers - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -############################################################################################## -################################ libFuzzer TESTS ############################################# -############################################################################################## - libFuzzerTest: - needs: [BuilderFuzzers] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/libfuzzer - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=libFuzzer tests - REPO_COPY=${{runner.temp}}/libfuzzer/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: libFuzzer test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 libfuzzer_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# ###################################### JEPSEN TESTS ######################################### ############################################################################################# Jepsen: From dd6f12dd94a93c916304ff9c0c0bd2dd2a40fcb9 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Sat, 16 Sep 2023 04:41:13 +0000 Subject: [PATCH 11/12] review suggestions --- .github/workflows/libfuzzer.yml | 21 +++++++-- .github/workflows/pull_request.yml | 13 ++++-- docker/test/libfuzzer/parse_options.py | 61 -------------------------- docker/test/libfuzzer/run_libfuzzer.py | 26 ++++++----- tests/ci/libfuzzer_test_check.py | 42 +++--------------- 5 files changed, 48 insertions(+), 115 deletions(-) delete mode 100755 docker/test/libfuzzer/parse_options.py diff --git a/.github/workflows/libfuzzer.yml b/.github/workflows/libfuzzer.yml index 74772ccf6d9..e8a0396684a 100644 --- a/.github/workflows/libfuzzer.yml +++ b/.github/workflows/libfuzzer.yml @@ -5,9 +5,9 @@ env: PYTHONUNBUFFERED: 1 on: # yamllint disable-line rule:truthy -# schedule: -# - cron: '0 0 2 31 1' # never for now - workflow_dispatch: + # schedule: + # - cron: '0 0 2 31 1' # never for now + workflow_call: jobs: BuilderFuzzers: runs-on: [self-hosted, builder] @@ -21,11 +21,19 @@ jobs: CACHES_PATH=${{runner.temp}}/../ccaches BUILD_NAME=fuzzers EOF + - name: Download changed images + # even if artifact does not exist, e.g. on `do not test` label or failed Docker job + continue-on-error: true + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true submodules: true + ref: ${{github.ref}} - name: Build run: | sudo rm -fr "$TEMP_PATH" @@ -57,6 +65,13 @@ jobs: REPO_COPY=${{runner.temp}}/libfuzzer/ClickHouse KILL_TIMEOUT=10800 EOF + - name: Download changed images + # even if artifact does not exist, e.g. on `do not test` label or failed Docker job + continue-on-error: true + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.TEMP_PATH }} - name: Download json reports uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ce135846dd5..838a6b56440 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -5186,9 +5186,16 @@ jobs: cd "$GITHUB_WORKSPACE/tests/ci" python3 finish_check.py python3 merge_pr.py --check-approved -############################################################################################## -########################### SQLLOGIC TEST ################################################### -############################################################################################## +############################################################################################# +####################################### libFuzzer ########################################### +############################################################################################# + libFuzzer: + if: contains(github.event.pull_request.labels.*.name, 'libFuzzer') + needs: [DockerHubPush, StyleCheck] + uses: ./.github/workflows/libfuzzer.yml + ############################################################################################## + ############################ SQLLOGIC TEST ################################################### + ############################################################################################## SQLLogicTestRelease: needs: [BuilderDebRelease] runs-on: [self-hosted, func-tester] diff --git a/docker/test/libfuzzer/parse_options.py b/docker/test/libfuzzer/parse_options.py deleted file mode 100755 index 5695e80a714..00000000000 --- a/docker/test/libfuzzer/parse_options.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -"""Helper script for parsing custom fuzzing options.""" -import configparser -import sys - - -def parse_options(options_file_path, options_section): - """Parses the given file and returns options from the given section.""" - parser = configparser.ConfigParser() - parser.read(options_file_path) - - if not parser.has_section(options_section): - return None - - options = parser[options_section] - - if options_section == "libfuzzer": - options_string = " ".join( - "-%s=%s" % (key, value) for key, value in options.items() - ) - else: - # Sanitizer options. - options_string = ":".join( - "%s=%s" % (key, value) for key, value in options.items() - ) - - return options_string - - -def main(): - """Processes the arguments and prints the options in the correct format.""" - if len(sys.argv) < 3: - sys.stderr.write( - "Usage: %s \n" % sys.argv[0] - ) - return 1 - - options = parse_options(sys.argv[1], sys.argv[2]) - if options is not None: - print(options) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/docker/test/libfuzzer/run_libfuzzer.py b/docker/test/libfuzzer/run_libfuzzer.py index b608c97de60..5ed019490d5 100755 --- a/docker/test/libfuzzer/run_libfuzzer.py +++ b/docker/test/libfuzzer/run_libfuzzer.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 +import configparser import logging import os from pathlib import Path import subprocess -from parse_options import parse_options DEBUGGER = os.getenv("DEBUGGER", "") FUZZER_ARGS = os.getenv("FUZZER_ARGS", "") @@ -23,25 +23,29 @@ def run_fuzzer(fuzzer: str): with Path(options_file) as path: if path.exists() and path.is_file(): - custom_asan_options = parse_options(options_file, "asan") - if custom_asan_options: + parser = configparser.ConfigParser() + parser.read(path) + + if parser.has_section("asan"): os.environ[ "ASAN_OPTIONS" - ] = f"{os.environ['ASAN_OPTIONS']}:{custom_asan_options}" + ] = f"{os.environ['ASAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['asan'].items())}" - custom_msan_options = parse_options(options_file, "msan") - if custom_msan_options: + if parser.has_section("msan"): os.environ[ "MSAN_OPTIONS" - ] = f"{os.environ['MSAN_OPTIONS']}:{custom_msan_options}" + ] = f"{os.environ['MSAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['msan'].items())}" - custom_ubsan_options = parse_options(options_file, "ubsan") - if custom_ubsan_options: + if parser.has_section("ubsan"): os.environ[ "UBSAN_OPTIONS" - ] = f"{os.environ['UBSAN_OPTIONS']}:{custom_ubsan_options}" + ] = f"{os.environ['UBSAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['ubsan'].items())}" - custom_libfuzzer_options = parse_options(options_file, "libfuzzer") + if parser.has_section("libfuzzer"): + custom_libfuzzer_options = " ".join( + "-%s=%s" % (key, value) + for key, value in parser["libfuzzer"].items() + ) cmd_line = f"{DEBUGGER} ./{fuzzer} {FUZZER_ARGS} {corpus_dir}" if custom_libfuzzer_options: diff --git a/tests/ci/libfuzzer_test_check.py b/tests/ci/libfuzzer_test_check.py index 9fee997cc96..8d307b22042 100644 --- a/tests/ci/libfuzzer_test_check.py +++ b/tests/ci/libfuzzer_test_check.py @@ -3,7 +3,6 @@ import argparse import logging import os -import subprocess import sys import atexit import zipfile @@ -99,59 +98,28 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("check_name") parser.add_argument("kill_timeout", type=int) - parser.add_argument( - "--validate-bugfix", - action="store_true", - help="Check that added tests failed on latest stable", - ) - parser.add_argument( - "--post-commit-status", - default="commit_status", - choices=["commit_status", "file"], - help="Where to public post commit status", - ) return parser.parse_args() -def docker_build_image(image_name: str, filepath: Path) -> DockerImage: - # context = filepath.parent - docker_image = DockerImage(image_name) - build_cmd = f"docker build --network=host -t {image_name} {filepath}" - logging.info("Will build image with cmd: '%s'", build_cmd) - subprocess.check_call( - build_cmd, - shell=True, - ) - return docker_image - - def main(): logging.basicConfig(level=logging.INFO) stopwatch = Stopwatch() - temp_path = TEMP_PATH - repo_path = REPO_COPY + temp_path = Path(TEMP_PATH) + repo_path = Path(REPO_COPY) reports_path = REPORTS_PATH args = parse_args() check_name = args.check_name kill_timeout = args.kill_timeout - validate_bugfix_check = args.validate_bugfix - run_changed_tests = validate_bugfix_check gh = Github(get_best_robot_token(), per_page=100) - - # For validate_bugfix_check we need up to date information about labels, so pr_event_from_api is used - pr_info = PRInfo( - need_changed_files=run_changed_tests, pr_event_from_api=validate_bugfix_check - ) - + pr_info = PRInfo() commit = get_commit(gh, pr_info.sha) atexit.register(update_mergeable_check, gh, pr_info, check_name) - if not Path(temp_path).exists(): - os.makedirs(temp_path) + temp_path.mkdir(parents=True, exist_ok=True) if "RUN_BY_HASH_NUM" in os.environ: run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM", "0")) @@ -201,7 +169,7 @@ def main(): run_command = get_run_command( fuzzers_path, - Path(repo_path), + repo_path, result_path, kill_timeout, additional_envs, From b1b49f430af91868d9da8cb0a34aa5fa58e0bc3b Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Tue, 19 Sep 2023 15:32:58 +0000 Subject: [PATCH 12/12] review suggestions --- docker/images.json | 1 + tests/ci/libfuzzer_test_check.py | 10 ++++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docker/images.json b/docker/images.json index bddfd49ea3b..d208ee9a888 100644 --- a/docker/images.json +++ b/docker/images.json @@ -125,6 +125,7 @@ "name": "clickhouse/test-base", "dependent": [ "docker/test/fuzzer", + "docker/test/libfuzzer", "docker/test/integration/base", "docker/test/keeper-jepsen", "docker/test/server-jepsen", diff --git a/tests/ci/libfuzzer_test_check.py b/tests/ci/libfuzzer_test_check.py index 8d307b22042..e768b7f1b4e 100644 --- a/tests/ci/libfuzzer_test_check.py +++ b/tests/ci/libfuzzer_test_check.py @@ -139,9 +139,8 @@ def main(): docker_image = get_image_with_version(reports_path, "clickhouse/libfuzzer") - fuzzers_path = Path(temp_path) / "fuzzers" - if not fuzzers_path.exists(): - os.makedirs(fuzzers_path) + fuzzers_path = temp_path / "fuzzers" + fuzzers_path.mkdir(parents=True, exist_ok=True) download_fuzzers(check_name, reports_path, fuzzers_path) @@ -152,9 +151,8 @@ def main(): corpus_path = fuzzers_path / (file.removesuffix("_seed_corpus.zip") + ".in") zipfile.ZipFile(fuzzers_path / file, "r").extractall(corpus_path) - result_path = Path(temp_path) / "result_path" - if not result_path.exists(): - os.makedirs(result_path) + result_path = temp_path / "result_path" + result_path.mkdir(parents=True, exist_ok=True) run_log_path = result_path / "run.log"