mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 23:21:59 +00:00
Merge pull request #54310 from ClickHouse/ci-libfuzzer-integration
CI: libFuzzer integration
This commit is contained in:
Mikhail f. Shiryaev
GitHub
commit
4882b99715
95
.github/workflows/libfuzzer.yml
vendored
Normal file
95
.github/workflows/libfuzzer.yml
vendored
Normal file
@ -0,0 +1,95 @@
|
||||
name: libFuzzer
|
||||
|
||||
env:
|
||||
# Force the stdout and stderr streams to be unbuffered
|
||||
PYTHONUNBUFFERED: 1
|
||||
|
||||
on: # yamllint disable-line rule:truthy
|
||||
# schedule:
|
||||
# - cron: '0 0 2 31 1' # never for now
|
||||
workflow_call:
|
||||
jobs:
|
||||
BuilderFuzzers:
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
- name: Set envs
|
||||
run: |
|
||||
cat >> "$GITHUB_ENV" << 'EOF'
|
||||
TEMP_PATH=${{runner.temp}}/build_check
|
||||
IMAGES_PATH=${{runner.temp}}/images_path
|
||||
REPO_COPY=${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH=${{runner.temp}}/../ccaches
|
||||
BUILD_NAME=fuzzers
|
||||
EOF
|
||||
- name: Download changed images
|
||||
# even if artifact does not exist, e.g. on `do not test` label or failed Docker job
|
||||
continue-on-error: true
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ env.IMAGES_PATH }}
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
with:
|
||||
clear-repository: true
|
||||
submodules: true
|
||||
ref: ${{github.ref}}
|
||||
- name: Build
|
||||
run: |
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
mkdir -p "$TEMP_PATH"
|
||||
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
|
||||
cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
|
||||
- name: Upload build URLs to artifacts
|
||||
if: ${{ success() || failure() }}
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ${{ env.BUILD_URLS }}
|
||||
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
|
||||
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
|
||||
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
|
||||
libFuzzerTest:
|
||||
needs: [BuilderFuzzers]
|
||||
runs-on: [self-hosted, func-tester]
|
||||
steps:
|
||||
- name: Set envs
|
||||
run: |
|
||||
cat >> "$GITHUB_ENV" << 'EOF'
|
||||
TEMP_PATH=${{runner.temp}}/libfuzzer
|
||||
REPORTS_PATH=${{runner.temp}}/reports_dir
|
||||
CHECK_NAME=libFuzzer tests
|
||||
REPO_COPY=${{runner.temp}}/libfuzzer/ClickHouse
|
||||
KILL_TIMEOUT=10800
|
||||
EOF
|
||||
- name: Download changed images
|
||||
# even if artifact does not exist, e.g. on `do not test` label or failed Docker job
|
||||
continue-on-error: true
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ env.TEMP_PATH }}
|
||||
- name: Download json reports
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
path: ${{ env.REPORTS_PATH }}
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
with:
|
||||
clear-repository: true
|
||||
- name: libFuzzer test
|
||||
run: |
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
mkdir -p "$TEMP_PATH"
|
||||
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
|
||||
cd "$REPO_COPY/tests/ci"
|
||||
python3 libfuzzer_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
|
||||
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
13
.github/workflows/pull_request.yml
vendored
13
.github/workflows/pull_request.yml
vendored
@ -5187,9 +5187,16 @@ jobs:
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
python3 finish_check.py
|
||||
python3 merge_pr.py --check-approved
|
||||
##############################################################################################
|
||||
########################### SQLLOGIC TEST ###################################################
|
||||
##############################################################################################
|
||||
#############################################################################################
|
||||
####################################### libFuzzer ###########################################
|
||||
#############################################################################################
|
||||
libFuzzer:
|
||||
if: contains(github.event.pull_request.labels.*.name, 'libFuzzer')
|
||||
needs: [DockerHubPush, StyleCheck]
|
||||
uses: ./.github/workflows/libfuzzer.yml
|
||||
##############################################################################################
|
||||
############################ SQLLOGIC TEST ###################################################
|
||||
##############################################################################################
|
||||
SQLLogicTestRelease:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, func-tester]
|
||||
|
||||
@ -19,6 +19,7 @@ include (cmake/tools.cmake)
|
||||
include (cmake/ccache.cmake)
|
||||
include (cmake/clang_tidy.cmake)
|
||||
include (cmake/git.cmake)
|
||||
include (cmake/utils.cmake)
|
||||
|
||||
# Ignore export() since we don't use it,
|
||||
# but it gets broken with a global targets via link_libraries()
|
||||
@ -562,22 +563,6 @@ add_subdirectory (programs)
|
||||
add_subdirectory (tests)
|
||||
add_subdirectory (utils)
|
||||
|
||||
# Function get_all_targets collects all targets recursively
|
||||
function(get_all_targets var)
|
||||
macro(get_all_targets_recursive targets dir)
|
||||
get_property(subdirectories DIRECTORY ${dir} PROPERTY SUBDIRECTORIES)
|
||||
foreach(subdir ${subdirectories})
|
||||
get_all_targets_recursive(${targets} ${subdir})
|
||||
endforeach()
|
||||
get_property(current_targets DIRECTORY ${dir} PROPERTY BUILDSYSTEM_TARGETS)
|
||||
list(APPEND ${targets} ${current_targets})
|
||||
endmacro()
|
||||
|
||||
set(targets)
|
||||
get_all_targets_recursive(targets ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
set(${var} ${targets} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
if (FUZZER)
|
||||
# Bundle fuzzers target
|
||||
add_custom_target(fuzzers)
|
||||
@ -592,14 +577,18 @@ if (FUZZER)
|
||||
# clickhouse fuzzer isn't working correctly
|
||||
# initial PR https://github.com/ClickHouse/ClickHouse/pull/27526
|
||||
#if (target MATCHES ".+_fuzzer" OR target STREQUAL "clickhouse")
|
||||
if (target MATCHES ".+_fuzzer")
|
||||
if (target_type STREQUAL "EXECUTABLE" AND target MATCHES ".+_fuzzer")
|
||||
message(STATUS "${target} instrumented with fuzzer")
|
||||
target_link_libraries(${target} PUBLIC ch_contrib::fuzzer)
|
||||
# Add to fuzzers bundle
|
||||
add_dependencies(fuzzers ${target})
|
||||
get_target_filename(${target} target_bin_name)
|
||||
get_target_property(target_bin_dir ${target} BINARY_DIR)
|
||||
add_custom_command(TARGET fuzzers POST_BUILD COMMAND mv "${target_bin_dir}/${target_bin_name}" "${CMAKE_CURRENT_BINARY_DIR}/programs/" VERBATIM)
|
||||
endif()
|
||||
endif()
|
||||
endforeach()
|
||||
add_custom_command(TARGET fuzzers POST_BUILD COMMAND SRC=${CMAKE_SOURCE_DIR} BIN=${CMAKE_BINARY_DIR} OUT=${CMAKE_BINARY_DIR}/programs ${CMAKE_SOURCE_DIR}/tests/fuzz/build.sh VERBATIM)
|
||||
endif()
|
||||
|
||||
include (cmake/sanitize_targets.cmake)
|
||||
|
||||
120
cmake/utils.cmake
Normal file
120
cmake/utils.cmake
Normal file
@ -0,0 +1,120 @@
|
||||
# Useful stuff
|
||||
|
||||
# Function get_all_targets collects all targets recursively
|
||||
function(get_all_targets outvar)
|
||||
macro(get_all_targets_recursive targets dir)
|
||||
get_property(subdirectories DIRECTORY ${dir} PROPERTY SUBDIRECTORIES)
|
||||
foreach(subdir ${subdirectories})
|
||||
get_all_targets_recursive(${targets} ${subdir})
|
||||
endforeach()
|
||||
get_property(current_targets DIRECTORY ${dir} PROPERTY BUILDSYSTEM_TARGETS)
|
||||
list(APPEND ${targets} ${current_targets})
|
||||
endmacro()
|
||||
|
||||
set(targets)
|
||||
get_all_targets_recursive(targets ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
set(${outvar} ${targets} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
|
||||
# Function get_target_filename calculates target's output file name
|
||||
function(get_target_filename target outvar)
|
||||
get_target_property(prop_type "${target}" TYPE)
|
||||
get_target_property(prop_is_framework "${target}" FRAMEWORK)
|
||||
get_target_property(prop_outname "${target}" OUTPUT_NAME)
|
||||
get_target_property(prop_archive_outname "${target}" ARCHIVE_OUTPUT_NAME)
|
||||
get_target_property(prop_library_outname "${target}" LIBRARY_OUTPUT_NAME)
|
||||
get_target_property(prop_runtime_outname "${target}" RUNTIME_OUTPUT_NAME)
|
||||
# message("prop_archive_outname: ${prop_archive_outname}")
|
||||
# message("prop_library_outname: ${prop_library_outname}")
|
||||
# message("prop_runtime_outname: ${prop_runtime_outname}")
|
||||
if(DEFINED CMAKE_BUILD_TYPE)
|
||||
get_target_property(prop_cfg_outname "${target}" "${OUTPUT_NAME}_${CMAKE_BUILD_TYPE}")
|
||||
get_target_property(prop_archive_cfg_outname "${target}" "${ARCHIVE_OUTPUT_NAME}_${CMAKE_BUILD_TYPE}")
|
||||
get_target_property(prop_library_cfg_outname "${target}" "${LIBRARY_OUTPUT_NAME}_${CMAKE_BUILD_TYPE}")
|
||||
get_target_property(prop_runtime_cfg_outname "${target}" "${RUNTIME_OUTPUT_NAME}_${CMAKE_BUILD_TYPE}")
|
||||
# message("prop_archive_cfg_outname: ${prop_archive_cfg_outname}")
|
||||
# message("prop_library_cfg_outname: ${prop_library_cfg_outname}")
|
||||
# message("prop_runtime_cfg_outname: ${prop_runtime_cfg_outname}")
|
||||
if(NOT ("${prop_cfg_outname}" STREQUAL "prop_cfg_outname-NOTFOUND"))
|
||||
set(prop_outname "${prop_cfg_outname}")
|
||||
endif()
|
||||
if(NOT ("${prop_archive_cfg_outname}" STREQUAL "prop_archive_cfg_outname-NOTFOUND"))
|
||||
set(prop_archive_outname "${prop_archive_cfg_outname}")
|
||||
endif()
|
||||
if(NOT ("${prop_library_cfg_outname}" STREQUAL "prop_library_cfg_outname-NOTFOUND"))
|
||||
set(prop_library_outname "${prop_library_cfg_outname}")
|
||||
endif()
|
||||
if(NOT ("${prop_runtime_cfg_outname}" STREQUAL "prop_runtime_cfg_outname-NOTFOUND"))
|
||||
set(prop_runtime_outname "${prop_runtime_cfg_outname}")
|
||||
endif()
|
||||
endif()
|
||||
set(outname "${target}")
|
||||
if(NOT ("${prop_outname}" STREQUAL "prop_outname-NOTFOUND"))
|
||||
set(outname "${prop_outname}")
|
||||
endif()
|
||||
if("${prop_is_framework}")
|
||||
set(filename "${outname}")
|
||||
elseif(prop_type STREQUAL "STATIC_LIBRARY")
|
||||
if(NOT ("${prop_archive_outname}" STREQUAL "prop_archive_outname-NOTFOUND"))
|
||||
set(outname "${prop_archive_outname}")
|
||||
endif()
|
||||
set(filename "${CMAKE_STATIC_LIBRARY_PREFIX}${outname}${CMAKE_STATIC_LIBRARY_SUFFIX}")
|
||||
elseif(prop_type STREQUAL "MODULE_LIBRARY")
|
||||
if(NOT ("${prop_library_outname}" STREQUAL "prop_library_outname-NOTFOUND"))
|
||||
set(outname "${prop_library_outname}")
|
||||
endif()
|
||||
set(filename "${CMAKE_SHARED_MODULE_LIBRARY_PREFIX}${outname}${CMAKE_SHARED_MODULE_LIBRARY_SUFFIX}")
|
||||
elseif(prop_type STREQUAL "SHARED_LIBRARY")
|
||||
if(WIN32)
|
||||
if(NOT ("${prop_runtime_outname}" STREQUAL "prop_runtime_outname-NOTFOUND"))
|
||||
set(outname "${prop_runtime_outname}")
|
||||
endif()
|
||||
else()
|
||||
if(NOT ("${prop_library_outname}" STREQUAL "prop_library_outname-NOTFOUND"))
|
||||
set(outname "${prop_library_outname}")
|
||||
endif()
|
||||
endif()
|
||||
set(filename "${CMAKE_SHARED_LIBRARY_PREFIX}${outname}${CMAKE_SHARED_LIBRARY_SUFFIX}")
|
||||
elseif(prop_type STREQUAL "EXECUTABLE")
|
||||
if(NOT ("${prop_runtime_outname}" STREQUAL "prop_runtime_outname-NOTFOUND"))
|
||||
set(outname "${prop_runtime_outname}")
|
||||
endif()
|
||||
set(filename "${CMAKE_EXECUTABLE_PREFIX}${outname}${CMAKE_EXECUTABLE_SUFFIX}")
|
||||
else()
|
||||
message(FATAL_ERROR "target \"${target}\" is not of type STATIC_LIBRARY, MODULE_LIBRARY, SHARED_LIBRARY, or EXECUTABLE.")
|
||||
endif()
|
||||
set("${outvar}" "${filename}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
|
||||
# Function get_cmake_properties returns list of all propreties that cmake supports
|
||||
function(get_cmake_properties outvar)
|
||||
execute_process(COMMAND cmake --help-property-list OUTPUT_VARIABLE cmake_properties)
|
||||
# Convert command output into a CMake list
|
||||
string(REGEX REPLACE ";" "\\\\;" cmake_properties "${cmake_properties}")
|
||||
string(REGEX REPLACE "\n" ";" cmake_properties "${cmake_properties}")
|
||||
list(REMOVE_DUPLICATES cmake_properties)
|
||||
set("${outvar}" "${cmake_properties}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
# Function get_target_property_list returns list of all propreties set for target
|
||||
function(get_target_property_list target outvar)
|
||||
get_cmake_properties(cmake_property_list)
|
||||
foreach(property ${cmake_property_list})
|
||||
string(REPLACE "<CONFIG>" "${CMAKE_BUILD_TYPE}" property ${property})
|
||||
|
||||
# https://stackoverflow.com/questions/32197663/how-can-i-remove-the-the-location-property-may-not-be-read-from-target-error-i
|
||||
if(property STREQUAL "LOCATION" OR property MATCHES "^LOCATION_" OR property MATCHES "_LOCATION$")
|
||||
continue()
|
||||
endif()
|
||||
|
||||
get_property(was_set TARGET ${target} PROPERTY ${property} SET)
|
||||
if(was_set)
|
||||
get_target_property(value ${target} ${property})
|
||||
string(REGEX REPLACE ";" "\\\\\\\\;" value "${value}")
|
||||
list(APPEND outvar "${property} = ${value}")
|
||||
endif()
|
||||
endforeach()
|
||||
set(${outvar} ${${outvar}} PARENT_SCOPE)
|
||||
endfunction()
|
||||
@ -21,6 +21,10 @@
|
||||
"name": "clickhouse/fuzzer",
|
||||
"dependent": []
|
||||
},
|
||||
"docker/test/libfuzzer": {
|
||||
"name": "clickhouse/libfuzzer",
|
||||
"dependent": []
|
||||
},
|
||||
"docker/test/performance-comparison": {
|
||||
"name": "clickhouse/performance-comparison",
|
||||
"dependent": []
|
||||
@ -121,6 +125,7 @@
|
||||
"name": "clickhouse/test-base",
|
||||
"dependent": [
|
||||
"docker/test/fuzzer",
|
||||
"docker/test/libfuzzer",
|
||||
"docker/test/integration/base",
|
||||
"docker/test/keeper-jepsen",
|
||||
"docker/test/server-jepsen",
|
||||
|
||||
@ -78,6 +78,7 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
|
||||
python3-boto3 \
|
||||
yasm \
|
||||
zstd \
|
||||
zip \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists
|
||||
|
||||
|
||||
@ -97,11 +97,10 @@ if [ -n "$MAKE_DEB" ]; then
|
||||
bash -x /build/packages/build
|
||||
fi
|
||||
|
||||
if [ "$BUILD_TARGET" != "fuzzers" ]; then
|
||||
mv ./programs/clickhouse* /output
|
||||
[ -x ./programs/self-extracting/clickhouse ] && mv ./programs/self-extracting/clickhouse /output
|
||||
mv ./src/unit_tests_dbms /output ||: # may not exist for some binary builds
|
||||
fi
|
||||
mv ./programs/clickhouse* /output || mv ./programs/*_fuzzer /output
|
||||
[ -x ./programs/self-extracting/clickhouse ] && mv ./programs/self-extracting/clickhouse /output
|
||||
mv ./src/unit_tests_dbms /output ||: # may not exist for some binary builds
|
||||
mv ./programs/*.dict ./programs/*.options ./programs/*_seed_corpus.zip /output ||: # libFuzzer oss-fuzz compatible infrastructure
|
||||
|
||||
prepare_combined_output () {
|
||||
local OUTPUT
|
||||
|
||||
43
docker/test/libfuzzer/Dockerfile
Normal file
43
docker/test/libfuzzer/Dockerfile
Normal file
@ -0,0 +1,43 @@
|
||||
ARG FROM_TAG=latest
|
||||
FROM clickhouse/test-base:$FROM_TAG
|
||||
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
ENV LANG=C.UTF-8
|
||||
ENV TZ=Europe/Amsterdam
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
|
||||
RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
|
||||
ca-certificates \
|
||||
libc6-dbg \
|
||||
moreutils \
|
||||
ncdu \
|
||||
p7zip-full \
|
||||
parallel \
|
||||
psmisc \
|
||||
python3 \
|
||||
python3-pip \
|
||||
rsync \
|
||||
tree \
|
||||
tzdata \
|
||||
vim \
|
||||
wget \
|
||||
&& apt-get autoremove --yes \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN pip3 install Jinja2
|
||||
|
||||
COPY * /
|
||||
|
||||
ENV FUZZER_ARGS="-max_total_time=60"
|
||||
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
CMD set -o pipefail \
|
||||
&& timeout -s 9 1h /run_libfuzzer.py 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log
|
||||
|
||||
# docker run --network=host --volume <workspace>:/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/libfuzzer
|
||||
|
||||
77
docker/test/libfuzzer/run_libfuzzer.py
Executable file
77
docker/test/libfuzzer/run_libfuzzer.py
Executable file
@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import configparser
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
DEBUGGER = os.getenv("DEBUGGER", "")
|
||||
FUZZER_ARGS = os.getenv("FUZZER_ARGS", "")
|
||||
|
||||
|
||||
def run_fuzzer(fuzzer: str):
|
||||
logging.info(f"Running fuzzer {fuzzer}...")
|
||||
|
||||
corpus_dir = f"{fuzzer}.in"
|
||||
with Path(corpus_dir) as path:
|
||||
if not path.exists() or not path.is_dir():
|
||||
corpus_dir = ""
|
||||
|
||||
options_file = f"{fuzzer}.options"
|
||||
custom_libfuzzer_options = ""
|
||||
|
||||
with Path(options_file) as path:
|
||||
if path.exists() and path.is_file():
|
||||
parser = configparser.ConfigParser()
|
||||
parser.read(path)
|
||||
|
||||
if parser.has_section("asan"):
|
||||
os.environ[
|
||||
"ASAN_OPTIONS"
|
||||
] = f"{os.environ['ASAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['asan'].items())}"
|
||||
|
||||
if parser.has_section("msan"):
|
||||
os.environ[
|
||||
"MSAN_OPTIONS"
|
||||
] = f"{os.environ['MSAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['msan'].items())}"
|
||||
|
||||
if parser.has_section("ubsan"):
|
||||
os.environ[
|
||||
"UBSAN_OPTIONS"
|
||||
] = f"{os.environ['UBSAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['ubsan'].items())}"
|
||||
|
||||
if parser.has_section("libfuzzer"):
|
||||
custom_libfuzzer_options = " ".join(
|
||||
"-%s=%s" % (key, value)
|
||||
for key, value in parser["libfuzzer"].items()
|
||||
)
|
||||
|
||||
cmd_line = f"{DEBUGGER} ./{fuzzer} {FUZZER_ARGS} {corpus_dir}"
|
||||
if custom_libfuzzer_options:
|
||||
cmd_line += f" {custom_libfuzzer_options}"
|
||||
|
||||
if not "-dict=" in cmd_line and Path(f"{fuzzer}.dict").exists():
|
||||
cmd_line += f" -dict={fuzzer}.dict"
|
||||
|
||||
cmd_line += " < /dev/null"
|
||||
|
||||
logging.info(f"...will execute: {cmd_line}")
|
||||
subprocess.check_call(cmd_line, shell=True)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
subprocess.check_call("ls -al", shell=True)
|
||||
|
||||
with Path() as current:
|
||||
for fuzzer in current.iterdir():
|
||||
if (current / fuzzer).is_file() and os.access(current / fuzzer, os.X_OK):
|
||||
run_fuzzer(fuzzer)
|
||||
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -210,3 +210,12 @@ def download_performance_build(check_name, reports_path, result_path):
|
||||
result_path,
|
||||
lambda x: x.endswith("performance.tar.zst"),
|
||||
)
|
||||
|
||||
|
||||
def download_fuzzers(check_name, reports_path, result_path):
|
||||
download_builds_filter(
|
||||
check_name,
|
||||
reports_path,
|
||||
result_path,
|
||||
lambda x: x.endswith(("_fuzzer", ".dict", ".options", "_seed_corpus.zip")),
|
||||
)
|
||||
|
||||
@ -282,6 +282,7 @@ CI_CONFIG = CiConfig(
|
||||
"SQLancer (debug)": TestConfig("package_debug"),
|
||||
"Sqllogic test (release)": TestConfig("package_release"),
|
||||
"SQLTest": TestConfig("package_release"),
|
||||
"libFuzzer tests": TestConfig("fuzzers"),
|
||||
},
|
||||
)
|
||||
CI_CONFIG.validate()
|
||||
|
||||
190
tests/ci/libfuzzer_test_check.py
Normal file
190
tests/ci/libfuzzer_test_check.py
Normal file
@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import atexit
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
from github import Github
|
||||
|
||||
from build_download_helper import download_fuzzers
|
||||
from clickhouse_helper import (
|
||||
CiLogsCredentials,
|
||||
)
|
||||
from commit_status_helper import (
|
||||
RerunHelper,
|
||||
get_commit,
|
||||
update_mergeable_check,
|
||||
)
|
||||
from docker_pull_helper import DockerImage, get_image_with_version
|
||||
|
||||
from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH
|
||||
from get_robot_token import get_best_robot_token
|
||||
from pr_info import PRInfo
|
||||
from report import TestResults
|
||||
|
||||
from stopwatch import Stopwatch
|
||||
|
||||
from tee_popen import TeePopen
|
||||
|
||||
|
||||
NO_CHANGES_MSG = "Nothing to run"
|
||||
|
||||
|
||||
def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total):
|
||||
result = []
|
||||
if "DatabaseReplicated" in check_name:
|
||||
result.append("USE_DATABASE_REPLICATED=1")
|
||||
if "DatabaseOrdinary" in check_name:
|
||||
result.append("USE_DATABASE_ORDINARY=1")
|
||||
if "wide parts enabled" in check_name:
|
||||
result.append("USE_POLYMORPHIC_PARTS=1")
|
||||
if "ParallelReplicas" in check_name:
|
||||
result.append("USE_PARALLEL_REPLICAS=1")
|
||||
if "s3 storage" in check_name:
|
||||
result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1")
|
||||
if "analyzer" in check_name:
|
||||
result.append("USE_NEW_ANALYZER=1")
|
||||
|
||||
if run_by_hash_total != 0:
|
||||
result.append(f"RUN_BY_HASH_NUM={run_by_hash_num}")
|
||||
result.append(f"RUN_BY_HASH_TOTAL={run_by_hash_total}")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_run_command(
|
||||
fuzzers_path: Path,
|
||||
repo_path: Path,
|
||||
result_path: Path,
|
||||
kill_timeout: int,
|
||||
additional_envs: List[str],
|
||||
ci_logs_args: str,
|
||||
image: DockerImage,
|
||||
) -> str:
|
||||
additional_options = ["--hung-check"]
|
||||
additional_options.append("--print-time")
|
||||
|
||||
additional_options_str = (
|
||||
'-e ADDITIONAL_OPTIONS="' + " ".join(additional_options) + '"'
|
||||
)
|
||||
|
||||
envs = [
|
||||
f"-e MAX_RUN_TIME={int(0.9 * kill_timeout)}",
|
||||
# a static link, don't use S3_URL or S3_DOWNLOAD
|
||||
'-e S3_URL="https://s3.amazonaws.com/clickhouse-datasets"',
|
||||
]
|
||||
|
||||
envs += [f"-e {e}" for e in additional_envs]
|
||||
|
||||
env_str = " ".join(envs)
|
||||
|
||||
return (
|
||||
f"docker run "
|
||||
f"{ci_logs_args} "
|
||||
f"--workdir=/fuzzers "
|
||||
f"--volume={fuzzers_path}:/fuzzers "
|
||||
f"--volume={repo_path}/tests:/usr/share/clickhouse-test "
|
||||
f"--volume={result_path}:/test_output "
|
||||
f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}"
|
||||
)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("check_name")
|
||||
parser.add_argument("kill_timeout", type=int)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
stopwatch = Stopwatch()
|
||||
|
||||
temp_path = Path(TEMP_PATH)
|
||||
repo_path = Path(REPO_COPY)
|
||||
reports_path = REPORTS_PATH
|
||||
|
||||
args = parse_args()
|
||||
check_name = args.check_name
|
||||
kill_timeout = args.kill_timeout
|
||||
|
||||
gh = Github(get_best_robot_token(), per_page=100)
|
||||
pr_info = PRInfo()
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
atexit.register(update_mergeable_check, gh, pr_info, check_name)
|
||||
|
||||
temp_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if "RUN_BY_HASH_NUM" in os.environ:
|
||||
run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM", "0"))
|
||||
run_by_hash_total = int(os.getenv("RUN_BY_HASH_TOTAL", "0"))
|
||||
check_name_with_group = (
|
||||
check_name + f" [{run_by_hash_num + 1}/{run_by_hash_total}]"
|
||||
)
|
||||
else:
|
||||
run_by_hash_num = 0
|
||||
run_by_hash_total = 0
|
||||
check_name_with_group = check_name
|
||||
|
||||
rerun_helper = RerunHelper(commit, check_name_with_group)
|
||||
if rerun_helper.is_already_finished_by_status():
|
||||
logging.info("Check is already finished according to github status, exiting")
|
||||
sys.exit(0)
|
||||
|
||||
docker_image = get_image_with_version(reports_path, "clickhouse/libfuzzer")
|
||||
|
||||
fuzzers_path = temp_path / "fuzzers"
|
||||
fuzzers_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
download_fuzzers(check_name, reports_path, fuzzers_path)
|
||||
|
||||
for file in os.listdir(fuzzers_path):
|
||||
if file.endswith("_fuzzer"):
|
||||
os.chmod(fuzzers_path / file, 0o777)
|
||||
elif file.endswith("_seed_corpus.zip"):
|
||||
corpus_path = fuzzers_path / (file.removesuffix("_seed_corpus.zip") + ".in")
|
||||
zipfile.ZipFile(fuzzers_path / file, "r").extractall(corpus_path)
|
||||
|
||||
result_path = temp_path / "result_path"
|
||||
result_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
run_log_path = result_path / "run.log"
|
||||
|
||||
additional_envs = get_additional_envs(
|
||||
check_name, run_by_hash_num, run_by_hash_total
|
||||
)
|
||||
|
||||
ci_logs_credentials = CiLogsCredentials(Path(temp_path) / "export-logs-config.sh")
|
||||
ci_logs_args = ci_logs_credentials.get_docker_arguments(
|
||||
pr_info, stopwatch.start_time_str, check_name
|
||||
)
|
||||
|
||||
run_command = get_run_command(
|
||||
fuzzers_path,
|
||||
repo_path,
|
||||
result_path,
|
||||
kill_timeout,
|
||||
additional_envs,
|
||||
ci_logs_args,
|
||||
docker_image,
|
||||
)
|
||||
logging.info("Going to run libFuzzer tests: %s", run_command)
|
||||
|
||||
with TeePopen(run_command, run_log_path) as process:
|
||||
retcode = process.wait()
|
||||
if retcode == 0:
|
||||
logging.info("Run successfully")
|
||||
else:
|
||||
logging.info("Run failed")
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -55,6 +55,7 @@ class TeePopen:
|
||||
stderr=STDOUT,
|
||||
stdout=PIPE,
|
||||
bufsize=1,
|
||||
errors="backslashreplace",
|
||||
)
|
||||
if self.timeout is not None and self.timeout > 0:
|
||||
t = Thread(target=self._check_timeout)
|
||||
|
||||
28
tests/fuzz/build.sh
Executable file
28
tests/fuzz/build.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/bin/bash -eu
|
||||
|
||||
# copy fuzzer options and dictionaries
|
||||
cp $SRC/tests/fuzz/*.dict $OUT/
|
||||
cp $SRC/tests/fuzz/*.options $OUT/
|
||||
|
||||
# prepare corpus dirs
|
||||
mkdir -p $BIN/tests/fuzz/lexer_fuzzer.in/
|
||||
mkdir -p $BIN/tests/fuzz/select_parser_fuzzer.in/
|
||||
mkdir -p $BIN/tests/fuzz/create_parser_fuzzer.in/
|
||||
mkdir -p $BIN/tests/fuzz/execute_query_fuzzer.in/
|
||||
|
||||
# prepare corpus
|
||||
cp $SRC/tests/queries/0_stateless/*.sql $BIN/tests/fuzz/lexer_fuzzer.in/
|
||||
cp $SRC/tests/queries/0_stateless/*.sql $BIN/tests/fuzz/select_parser_fuzzer.in/
|
||||
cp $SRC/tests/queries/0_stateless/*.sql $BIN/tests/fuzz/create_parser_fuzzer.in/
|
||||
cp $SRC/tests/queries/0_stateless/*.sql $BIN/tests/fuzz/execute_query_fuzzer.in/
|
||||
cp $SRC/tests/queries/1_stateful/*.sql $BIN/tests/fuzz/lexer_fuzzer.in/
|
||||
cp $SRC/tests/queries/1_stateful/*.sql $BIN/tests/fuzz/select_parser_fuzzer.in/
|
||||
cp $SRC/tests/queries/1_stateful/*.sql $BIN/tests/fuzz/create_parser_fuzzer.in/
|
||||
cp $SRC/tests/queries/1_stateful/*.sql $BIN/tests/fuzz/execute_query_fuzzer.in/
|
||||
|
||||
# build corpus archives
|
||||
cd $BIN/tests/fuzz
|
||||
for dir in *_fuzzer.in; do
|
||||
fuzzer=$(basename $dir .in)
|
||||
zip -rj "$OUT/${fuzzer}_seed_corpus.zip" "${dir}/"
|
||||
done
|
||||
Loading…
Reference in New Issue
Block a user