[fuzz] add fuzz coverage CI check

.azure-pipelines/pipelines.yml

@@ -90,21 +90,28 @@ jobs:
           ciTarget: $(CI_TARGET)
 
   - job: coverage
-    displayName: "Linux-x64 coverage"
+    displayName: "Linux-x64"
     dependsOn: ["format"]
     timeoutInMinutes: 360
     pool: "x64-large"
+    strategy:
+      maxParallel: 2
+      matrix:
+        coverage:
+          CI_TARGET: "coverage"
+        fuzz_coverage:
+          CI_TARGET: "fuzz_coverage"
     steps:
       - template: bazel.yml
         parameters:
           managedAgent: false
-          ciTarget: bazel.coverage
+          ciTarget: bazel.$(CI_TARGET)
           rbe: false
           # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces
           bazelBuildExtraOptions: "--test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base"
 
-      - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/coverage coverage'
-        displayName: "Upload Report to GCS"
+      - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/$(CI_TARGET) $(CI_TARGET)'
+        displayName: "Upload $(CI_TARGET) Report to GCS"
         env:
           ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
           GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)

.bazelrc

@@ -234,18 +234,22 @@ build:remote-ci --remote_cache=grpcs://remotebuildexecution.googleapis.com
 build:remote-ci --remote_executor=grpcs://remotebuildexecution.googleapis.com
 
 # Fuzz builds
-build:asan-fuzzer --config=clang-asan
+# -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION is passed in in the bazel build target
+# rules for fuzz tests. Passing it in the CLI will cause dependencies to be build
+# with the macro. Causing issues in RouteMatcherTest.TestRoutes that expect prod
+# behavior from RE2 library.
+build:asan-fuzzer --config=asan
 build:asan-fuzzer --define=FUZZING_ENGINE=libfuzzer
 build:asan-fuzzer --copt=-fsanitize=fuzzer-no-link
 build:asan-fuzzer --copt=-fno-omit-frame-pointer
-build:asan-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
 # Remove UBSAN halt_on_error to avoid crashing on protobuf errors.
 build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1
 
 # Fuzzing without ASAN. This is useful for profiling fuzzers without any ASAN artifacts.
 build:plain-fuzzer --define=FUZZING_ENGINE=libfuzzer
 build:plain-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
 build:plain-fuzzer --copt=-fsanitize=fuzzer-no-link
+build:plain-fuzzer --define ENVOY_CONFIG_ASAN=1
 
 # Compile database generation config
 # We don't care about built binaries so always strip and use fastbuild.

bazel/README.md

@@ -638,13 +638,19 @@ test/run_envoy_bazel_coverage.sh
 The summary results are printed to the standard output and the full coverage
 report is available in `generated/coverage/coverage.html`.
 
+To generate coverage results for fuzz targets, use the `FUZZ_COVERAGE` environment variable, e.g.:
+```
+FUZZ_COVERAGE=true VALIDATE_COVERAGE=false test/run_envoy_bazel_coverage.sh
+```
+This generates a coverage report for fuzz targets after running the target for one minute against fuzzing engine libfuzzer using its coprus as initial seed inputs. The full coverage report will be available in `generated/fuzz_coverage/coverage.html`.
+
 Coverage for every PR is available in Circle in the "artifacts" tab of the coverage job. You will
 need to navigate down and open "coverage.html" but then you can navigate per normal. NOTE: We
 have seen some issues with seeing the artifacts tab. If you can't see it, log out of Circle, and
 then log back in and it should start working.
 
 The latest coverage report for master is available
-[here](https://storage.googleapis.com/envoy-postsubmit/master/coverage/index.html).
+[here](https://storage.googleapis.com/envoy-postsubmit/master/coverage/index.html). The latest fuzz coverage report for master is available [here](https://storage.googleapis.com/envoy-postsubmit/master/fuzz_coverage/index.html).
 
 It's also possible to specialize the coverage build to a specified test or test dir. This is useful
 when doing things like exploring the coverage of a fuzzer over its corpus. This can be done by

bazel/coverage/fuzz_coverage_wrapper.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -ex
+set -x
 
 TEST_BINARY=$1
 shift
@@ -11,4 +11,7 @@ rm -rf fuzz_corpus
 mkdir -p fuzz_corpus/seed_corpus
 cp -r $@ fuzz_corpus/seed_corpus
 
-${TEST_BINARY} fuzz_corpus -seed=${FUZZ_CORPUS_SEED:-1} -max_total_time=${FUZZ_CORPUS_TIME:-60}
+# TODO(asraa): When fuzz targets are stable, remove error suppression and run coverage while fuzzing.
+LLVM_PROFILE_FILE= ${TEST_BINARY} fuzz_corpus -seed=${FUZZ_CORPUS_SEED:-1} -max_total_time=${FUZZ_CORPUS_TIME:-60} -max_len=2048 || true
+
+${TEST_BINARY} fuzz_corpus -runs=0

ci/build_setup.sh

@@ -105,6 +105,9 @@ mkdir -p "${ENVOY_DELIVERY_DIR}"
 # This is where we copy the coverage report to.
 export ENVOY_COVERAGE_ARTIFACT="${ENVOY_BUILD_DIR}"/generated/coverage.tar.gz
 
+# This is where we copy the fuzz coverage report to.
+export ENVOY_FUZZ_COVERAGE_ARTIFACT="${ENVOY_BUILD_DIR}"/generated/fuzz_coverage.tar.gz
+
 # This is where we dump failed test logs for CI collection.
 export ENVOY_FAILED_TEST_LOGS="${ENVOY_BUILD_DIR}"/generated/failed-testlogs
 mkdir -p "${ENVOY_FAILED_TEST_LOGS}"

ci/do_ci.sh

@@ -269,6 +269,18 @@ elif [[ "$CI_TARGET" == "bazel.coverage" ]]; then
   test/run_envoy_bazel_coverage.sh ${COVERAGE_TEST_TARGETS}
   collect_build_profile coverage
   exit 0
+elif [[ "$CI_TARGET" == "bazel.fuzz_coverage" ]]; then
+  setup_clang_toolchain
+  echo "bazel coverage build with fuzz tests ${COVERAGE_TEST_TARGETS}"
+
+  # Reduce the amount of memory Bazel tries to use to prevent it from launching too many subprocesses.
+  # This should prevent the system from running out of memory and killing tasks. See discussion on
+  # https://github.com/envoyproxy/envoy/pull/5611.
+  [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --local_ram_resources=12288"
+
+  FUZZ_COVERAGE=true test/run_envoy_bazel_coverage.sh ${COVERAGE_TEST_TARGETS}
+  collect_build_profile coverage
+  exit 0
 elif [[ "$CI_TARGET" == "bazel.clang_tidy" ]]; then
   setup_clang_toolchain
   NUM_CPUS=$NUM_CPUS ci/run_clang_tidy.sh

test/fuzz/README.md

@@ -158,7 +158,15 @@ to provide fuzzers some interesting starting points for invalid inputs.
 ## Coverage reports
 
 Coverage reports, where individual lines are annotated with fuzzing hit counts, are a useful way to
-understand the scope and efficacy of the Envoy fuzzers. You can generate such reports from the
+understand the scope and efficacy of the Envoy fuzzers. You can generate fuzz coverage reports both locally, and using the OSS-Fuzz infrastructure.
+
+To generate fuzz coverage reports locally (see [Coverage builds](bazel/README.md), run
+```
+FUZZ_COVERAGE=true test/run_envoy_bazel_coverage.sh
+```
+This generates a coverage report after running the fuzz targets for one minute against the fuzzing engine libfuzzer and using the checked-in corpus as an initial seed.
+
+Otherwise, you can generate reports from the
 ClusterFuzz corpus following the general ClusterFuzz [instructions for profiling
 setup](https://github.com/google/oss-fuzz/blob/master/docs/code_coverage.md).
 

test/run_envoy_bazel_coverage.sh

@@ -20,10 +20,14 @@ elif [[ -n "${COVERAGE_TARGET}" ]]; then
   COVERAGE_TARGETS=${COVERAGE_TARGET}
 else
   # For fuzz builds, this overrides to just fuzz targets.
-  COVERAGE_TARGETS=//test/... && [[ ${FUZZ_COVERAGE} == "true" ]] &&
-    COVERAGE_TARGETS="$(bazel query 'attr("tags", "fuzz_target", //test/...)')"
+  COVERAGE_TARGETS=//test/...
 fi
 
+if [[ "${FUZZ_COVERAGE}" == "true" ]]; then
+  COVERAGE_TARGETS=$(bazel query "attr("tags", "fuzz_target", ${COVERAGE_TARGETS})")
+fi
+echo "COVERAGE_TARGETS=${COVERAGE_TARGETS}"
+
 if [[ "${FUZZ_COVERAGE}" == "true" ]]; then
   BAZEL_BUILD_OPTIONS+=" --config=fuzz-coverage --test_tag_filters=-nocoverage"
 else
@@ -36,7 +40,7 @@ bazel coverage ${BAZEL_BUILD_OPTIONS} ${COVERAGE_TARGETS}
 [[ -z "${ENVOY_BUILD_PROFILE}" ]] || cp -f "$(bazel info output_base)/command.profile.gz" "${ENVOY_BUILD_PROFILE}/coverage.profile.gz" || true
 [[ -z "${ENVOY_BUILD_DIR}" ]] || find bazel-testlogs/ -name test.log | tar zcf "${ENVOY_BUILD_DIR}/testlogs.tar.gz" -T -
 
-COVERAGE_DIR="${SRCDIR}"/generated/coverage
+COVERAGE_DIR="${SRCDIR}"/generated/coverage && [[ ${FUZZ_COVERAGE} == "true" ]] && COVERAGE_DIR="${SRCDIR}"/generated/fuzz_coverage
 
 rm -rf "${COVERAGE_DIR}"
 mkdir -p "${COVERAGE_DIR}"
@@ -47,7 +51,12 @@ cp bazel-out/_coverage/_coverage_report.dat "${COVERAGE_DATA}"
 COVERAGE_VALUE=$(genhtml --prefix ${PWD} --output "${COVERAGE_DIR}" "${COVERAGE_DATA}" | tee /dev/stderr | grep lines... | cut -d ' ' -f 4)
 COVERAGE_VALUE=${COVERAGE_VALUE%?}
 
-[[ -z "${ENVOY_COVERAGE_ARTIFACT}" ]] || tar zcf "${ENVOY_COVERAGE_ARTIFACT}" -C ${COVERAGE_DIR} --transform 's/^\./coverage/' .
+if [ "${FUZZ_COVERAGE}" == "true" ]
+then
+  [[ -z "${ENVOY_FUZZ_COVERAGE_ARTIFACT}" ]] || tar zcf "${ENVOY_FUZZ_COVERAGE_ARTIFACT}" -C ${COVERAGE_DIR} --transform 's/^\./fuzz_coverage/' .
+else
+  [[ -z "${ENVOY_COVERAGE_ARTIFACT}" ]] || tar zcf "${ENVOY_COVERAGE_ARTIFACT}" -C ${COVERAGE_DIR} --transform 's/^\./coverage/' .
+fi
 
 if [[ "$VALIDATE_COVERAGE" == "true" ]]; then
   if [[ "${FUZZ_COVERAGE}" == "true" ]]; then
@@ -66,7 +75,7 @@ fi
 
 # We want to allow per_file_coverage to fail without exiting this script.
 set +e
-if [[ "$VALIDATE_COVERAGE" == "true" ]]; then
+if [[ "$VALIDATE_COVERAGE" == "true" ]] && [[ "{FUZZ_COVERAGE}" == "false"]]; then
   echo "Checking per-extension coverage"
   output=$(./test/per_file_coverage.sh)
 

test/server/BUILD

@@ -322,7 +322,6 @@ envoy_cc_fuzz_test(
         "//test/integration:integration_lib",
         "//test/mocks/server:options_mocks",
         "//test/mocks/server:hot_restart_mocks",
-        "//test/mocks/stats:stats_mocks",
         "//test/test_common:environment_lib",
     ] + select({
         "//bazel:windows_x86_64": envoy_all_extensions(WINDOWS_SKIP_TARGETS),