diff --git a/bazel/envoy_test.bzl b/bazel/envoy_test.bzl index ca0b430c16f0b..222cb99e60e5d 100644 --- a/bazel/envoy_test.bzl +++ b/bazel/envoy_test.bzl @@ -121,7 +121,7 @@ def envoy_cc_fuzz_test( ], }), size = size, - tags = tags, + tags = ["fuzz_target"] + tags, ) # This target exists only for diff --git a/test/build_and_run_fuzz_targets.sh b/test/build_and_run_fuzz_targets.sh new file mode 100755 index 0000000000000..516b85fe1a6a3 --- /dev/null +++ b/test/build_and_run_fuzz_targets.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +if [[ $# -gt 0 ]]; then + FUZZ_TARGETS=$* +else + echo "This script should be called from tools/run_envoy_bazel_coverage.sh" +fi + +LIBFUZZER_TARGETS="" +# Build all fuzz targets to run instrumented with libfuzzer in sequence. +for t in ${FUZZ_TARGETS} +do + LIBFUZZER_TARGETS+="${t}_with_libfuzzer " +done + +bazel build ${BAZEL_BUILD_OPTIONS} ${LIBFUZZER_TARGETS} --config asan-fuzzer -c opt + +# Now run each fuzz target in parallel for 60 seconds. +PIDS="" +TMPDIR="${FUZZ_TEMPDIR}" + +for t in ${FUZZ_TARGETS} +do + # Make a temporary corpus for this fuzz target. + TARGET_BINARY="${t/://}" + TEMP_CORPUS_PATH="${TARGET_BINARY:2}" + CORPUS_DIR="${TMPDIR}/${TEMP_CORPUS_PATH////_}_corpus" + mkdir -v "${CORPUS_DIR}" + # Get the original corpus for the fuzz target + CORPUS_LOCATION="$(bazel query "labels(data, ${t})" | head -1)" + ORIGINAL_CORPUS="$(bazel query "labels(srcs, ${CORPUS_LOCATION})" | head -1)" + ORIGINAL_CORPUS="${ORIGINAL_CORPUS/://}" + ORIGINAL_CORPUS="$(dirname ${ORIGINAL_CORPUS})" + # Copy entries in original corpus into temp. + cp -r "$(pwd)${ORIGINAL_CORPUS:1}" "${CORPUS_DIR}" + # Run fuzzing process. + bazel-bin/"${TARGET_BINARY:2}"_with_libfuzzer -max_total_time=60 "${CORPUS_DIR}" & + # Add pid to pids list + PIDS="${PIDS} $!" +done + +# Wait for background process to run. +for pid in ${PIDS}; do + wait $pid + if [ $? -ne 0 ]; then + echo "${pid} FAILED" + fi +done diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 825803f28f97f..6eaf8c469a5a3 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -4,11 +4,13 @@ set -e [[ -z "${SRCDIR}" ]] && SRCDIR="${PWD}" [[ -z "${VALIDATE_COVERAGE}" ]] && VALIDATE_COVERAGE=true +[[ -z "${FUZZ_COVERAGE}" ]] && FUZZ_COVERAGE=false echo "Starting run_envoy_bazel_coverage.sh..." echo " PWD=$(pwd)" echo " SRCDIR=${SRCDIR}" echo " VALIDATE_COVERAGE=${VALIDATE_COVERAGE}" +echo " FUZZ_COVERAGE=${FUZZ_COVERAGE}" # This is the target that will be run to generate coverage data. It can be overridden by consumer # projects that want to run coverage on a different/combined target. @@ -18,18 +20,56 @@ if [[ $# -gt 0 ]]; then elif [[ -n "${COVERAGE_TARGET}" ]]; then COVERAGE_TARGETS=${COVERAGE_TARGET} else - COVERAGE_TARGETS=//test/... + # For fuzz builds, this overrides to just fuzz targets. + COVERAGE_TARGETS=//test/... && [[ ${FUZZ_COVERAGE} == "true" ]] && + COVERAGE_TARGETS="$(bazel query 'attr("tags", "fuzz_target", //test/...)')" fi -# Make sure //test/coverage:coverage_tests is up-to-date. SCRIPT_DIR="$(realpath "$(dirname "$0")")" -"${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS} +TEMP_CORPORA="" +if [ "$FUZZ_COVERAGE" == "true" ] +then + # Build and run libfuzzer linked target, grab collect temp directories. + FUZZ_TEMPDIR="$(mktemp -d)" + FUZZ_TEMPDIR=${FUZZ_TEMPDIR} "${SCRIPT_DIR}"/build_and_run_fuzz_targets.sh ${COVERAGE_TARGETS} +else + # Make sure //test/coverage:coverage_tests is up-to-date. + "${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS} +fi + +# Set the bazel targets to run. +BAZEL_TARGET=//test/coverage:coverage_tests && [[ ${FUZZ_COVERAGE} == "true" ]] && BAZEL_TARGET=${COVERAGE_TARGETS} + +# Add binaries to OBJECTS to pass in to llvm-cov +OBJECTS="" +# For nornaml builds, BAZEL_TARGET only contains //test/coverage:coverage_tests +for t in ${BAZEL_TARGET} +do + # Set test args. If normal coverage run, this is --log-path /dev/null + if [ "$FUZZ_COVERAGE" == "true" ] + then + # If this is a fuzz target, set args to be the temp corpus. + TARGET_BINARY="${t/://}" + CORPUS_LOCATION="${TARGET_BINARY:2}" + TEST_ARGS=(--test_arg="${FUZZ_TEMPDIR}/${CORPUS_LOCATION////_}_corpus" --test_arg="-runs=0") + if [[ -z "${OBJECTS}" ]]; then + # The first object needs to be passed without -object= flag. + OBJECTS="bazel-bin/${TARGET_BINARY:2}_with_libfuzzer" + else + OBJECTS="$OBJECTS -object=bazel-bin/${TARGET_BINARY:2}_with_libfuzzer" + fi + TARGET="${t}_with_libfuzzer" + else + TEST_ARGS=(--test_arg="--log-path /dev/null" --test_arg="-l trace") + OBJECTS="bazel-bin/test/coverage/coverage_tests" + TARGET="${t}" + fi -BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD_OPTIONS} \ + BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD_OPTIONS} \ -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... \ --test_timeout=2000 --cxxopt="-DENVOY_CONFIG_COVERAGE=1" --test_output=errors \ - --test_arg="--log-path /dev/null" --test_arg="-l trace" --test_env=HEAPCHECK= \ - //test/coverage:coverage_tests + "${TEST_ARGS[@]}" --test_env=HEAPCHECK= ${TARGET} +done COVERAGE_DIR="${SRCDIR}"/generated/coverage mkdir -p "${COVERAGE_DIR}" @@ -39,10 +79,11 @@ COVERAGE_BINARY="bazel-bin/test/coverage/coverage_tests" COVERAGE_DATA="${COVERAGE_DIR}/coverage.dat" echo "Merging coverage data..." -llvm-profdata merge -sparse -o ${COVERAGE_DATA} $(find -L bazel-out/k8-fastbuild/testlogs/test/coverage/coverage_tests/ -name coverage.dat) +BAZEL_OUT=test/coverage/coverage_tests/ && [[ ${FUZZ_COVERAGE} ]] && BAZEL_OUT=test/ +llvm-profdata merge -sparse -o ${COVERAGE_DATA} $(find -L bazel-out/k8-fastbuild/testlogs/${BAZEL_OUT} -name coverage.dat) echo "Generating report..." -llvm-cov show "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" -Xdemangler=c++filt \ +llvm-cov show -instr-profile="${COVERAGE_DATA}" ${OBJECTS} -Xdemangler=c++filt \ -ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" -output-dir=${COVERAGE_DIR} -format=html sed -i -e 's|>proc/self/cwd/|>|g' "${COVERAGE_DIR}/index.html" sed -i -e 's|>bazel-out/[^/]*/bin/\([^/]*\)/[^<]*/_virtual_includes/[^/]*|>\1|g' "${COVERAGE_DIR}/index.html" @@ -51,7 +92,7 @@ sed -i -e 's|>bazel-out/[^/]*/bin/\([^/]*\)/[^<]*/_virtual_includes/[^/]*|>\1|g' if [ "$VALIDATE_COVERAGE" == "true" ] then - COVERAGE_VALUE=$(llvm-cov export "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" \ + COVERAGE_VALUE=$(llvm-cov export "${OBJECTS}" -instr-profile="${COVERAGE_DATA}" \ -ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" -summary-only | \ python3 -c "import sys, json; print(json.load(sys.stdin)['data'][0]['totals']['lines']['percent'])") COVERAGE_THRESHOLD=97.0