[fuzz] add fuzz coverage CI check

.azure-pipelines/pipelines.yml

@@ -94,16 +94,31 @@ jobs:
     dependsOn: ["format"]
     timeoutInMinutes: 360
     pool: "x64-large"
+    strategy:
+      maxParallel: 2
+      matrix:
+        coverage:
+          CI_TARGET: "bazel.coverage"
+        fuzz_coverage:
+          CI_TARGET: "bazel.fuzz_coverage"
     steps:
       - template: bazel.yml
         parameters:
           managedAgent: false
-          ciTarget: bazel.coverage
+          ciTarget: $(CI_TARGET)
           rbe: ""
           bazelBuildExtraOptions: "--test_env=ENVOY_IP_TEST_VERSIONS=v4only --curses=no"
 
       - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /build/envoy/generated/coverage coverage'
-        displayName: "Upload Report to GCS"
+        displayName: "Upload Coverage Report to GCS"
+        env:
+          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+          GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+          GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket)
+        condition: always()
+
+      - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /build/envoy/generated/fuzz_coverage fuzz_coverage'
+        displayName: "Upload Fuzz Coverage Report to GCS"
         env:
           ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
           GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)

.bazelrc

@@ -209,11 +209,14 @@ build:remote-ci --remote_cache=grpcs://remotebuildexecution.googleapis.com
 build:remote-ci --remote_executor=grpcs://remotebuildexecution.googleapis.com
 
 # Fuzz builds
+# -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION is passed in in the bazel build target
+# rules for fuzz tests. Passing it in the CLI will cause dependencies to be build
+# with the macro. Causing issues in RouteMatcherTest.TestRoutes that expect prod
+# behavior from RE2 library.
 build:asan-fuzzer --config=clang-asan
 build:asan-fuzzer --define=FUZZING_ENGINE=libfuzzer
 build:asan-fuzzer --copt=-fsanitize=fuzzer-no-link
 build:asan-fuzzer --copt=-fno-omit-frame-pointer
-build:asan-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
 # Remove UBSAN halt_on_error to avoid crashing on protobuf errors.
 build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1
 

bazel/README.md

@@ -571,21 +571,31 @@ test/run_envoy_bazel_coverage.sh
 The summary results are printed to the standard output and the full coverage
 report is available in `generated/coverage/coverage.html`.
 
+To generate coverage results for fuzz targets, use the `FUZZ_COVERAGE` environment variable, e.g.:
+```
+FUZZ_COVERAGE=true VALIDATE_COVERAGE=false test/run_envoy_bazel_coverage.sh
+```
+This generates a coverage report for fuzz targets after running the target for one minute against fuzzing engine libfuzzer using its coprus as initial seed inputs. The full coverage report will be available in `generated/fuzz_coverage/coverage.html`.
+
 Coverage for every PR is available in Circle in the "artifacts" tab of the coverage job. You will
 need to navigate down and open "coverage.html" but then you can navigate per normal. NOTE: We
 have seen some issues with seeing the artifacts tab. If you can't see it, log out of Circle, and
 then log back in and it should start working.
 
 The latest coverage report for master is available
-[here](https://storage.googleapis.com/envoy-coverage/report-master/index.html).
+[here](https://storage.googleapis.com/envoy-coverage/report-master/index.html). The latest fuzz coverage report for master is available [here](https://storage.googleapis.com/envoy-fuzz-coverage/report-master/index.html)
 
 It's also possible to specialize the coverage build to a specified test or test dir. This is useful
 when doing things like exploring the coverage of a fuzzer over its corpus. This can be done by
 passing coverage targets as the command-line arguments and using the `VALIDATE_COVERAGE` environment
 variable, e.g.:
 
 ```
-VALIDATE_COVERAGE=false test/run_envoy_bazel_coverage.sh //test/common/common:base64_fuzz_test
+VALIDATE_COVERAGE=false test/run_envoy_bazel_coverage.sh //test/common/common:base64_test
+```
+For fuzz targets, you will need to include the `FUZZ_COVERAGE` environment variable, e.g.:
+```
+FUZZ_COVERAGE=true VALIDATE_COVERAGE=false test/run_envoy_bazel_coverage.sh //test/common/common:base64_fuzz_test
 ```
 
 # Cleaning the build and test artifacts

ci/build_setup.sh

@@ -106,6 +106,10 @@ mkdir -p "${ENVOY_DELIVERY_DIR}"
 export ENVOY_COVERAGE_DIR="${ENVOY_BUILD_DIR}"/generated/coverage
 mkdir -p "${ENVOY_COVERAGE_DIR}"
 
+# This is where we copy the coverage report to.
+export ENVOY_FUZZ_COVERAGE_DIR="${ENVOY_BUILD_DIR}"/generated/fuzz_coverage
+mkdir -p "${ENVOY_FUZZ_COVERAGE_DIR}"
+
 # This is where we dump failed test logs for CI collection.
 export ENVOY_FAILED_TEST_LOGS="${ENVOY_BUILD_DIR}"/generated/failed-testlogs
 mkdir -p "${ENVOY_FAILED_TEST_LOGS}"

ci/do_ci.sh

@@ -272,6 +272,18 @@ elif [[ "$CI_TARGET" == "bazel.coverage" ]]; then
   test/run_envoy_bazel_coverage.sh ${COVERAGE_TEST_TARGETS}
   collect_build_profile coverage
   exit 0
+elif [[ "$CI_TARGET" == "bazel.fuzz_coverage" ]]; then
+  setup_clang_toolchain
+  echo "bazel coverage build with fuzz tests ${COVERAGE_TEST_TARGETS}"
+
+  # Reduce the amount of memory Bazel tries to use to prevent it from launching too many subprocesses.
+  # This should prevent the system from running out of memory and killing tasks. See discussion on
+  # https://github.com/envoyproxy/envoy/pull/5611.
+  [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --local_ram_resources=12288"
+
+  FUZZ_COVERAGE=true test/run_envoy_bazel_coverage.sh ${COVERAGE_TEST_TARGETS}
+  collect_build_profile coverage
+  exit 0
 elif [[ "$CI_TARGET" == "bazel.clang_tidy" ]]; then
   setup_clang_toolchain
   NUM_CPUS=$NUM_CPUS ci/run_clang_tidy.sh

test/fuzz/README.md

@@ -158,7 +158,15 @@ to provide fuzzers some interesting starting points for invalid inputs.
 ## Coverage reports
 
 Coverage reports, where individual lines are annotated with fuzzing hit counts, are a useful way to
-understand the scope and efficacy of the Envoy fuzzers. You can generate such reports from the
+understand the scope and efficacy of the Envoy fuzzers. You can generate fuzz coverage reports both locally, and using the OSS-Fuzz infrastructure.
+
+To generate fuzz coverage reports locally (see [Coverage builds](bazel/README.md), run
+```
+FUZZ_COVERAGE=true test/run_envoy_bazel_coverage.sh
+```
+This generates a coverage report after running the fuzz targets for one minute against the fuzzing engine libfuzzer and using the checked-in corpus as an initial seed.
+
+Otherwise, you can generate reports from the
 ClusterFuzz corpus following the general ClusterFuzz [instructions for profiling
 setup](https://github.com/google/oss-fuzz/blob/master/docs/code_coverage.md).
 

test/run_envoy_bazel_coverage.sh

@@ -19,9 +19,12 @@ if [[ $# -gt 0 ]]; then
 elif [[ -n "${COVERAGE_TARGET}" ]]; then
   COVERAGE_TARGETS=${COVERAGE_TARGET}
 else
-  # For fuzz builds, this overrides to just fuzz targets.
-  COVERAGE_TARGETS=//test/... && [[ ${FUZZ_COVERAGE} == "true" ]] &&
-    COVERAGE_TARGETS="$(bazel query 'attr("tags", "fuzzer", //test/...)')"
+  COVERAGE_TARGETS=//test/...
+fi
+
+# For fuzz builds, filter out only libFuzzer linked targets.
+if [ "$FUZZ_COVERAGE" == "true" ]; then
+  COVERAGE_TARGETS="$(bazel query ${BAZEL_QUERY_OPTIONS} "attr('tags', 'fuzzer', ${COVERAGE_TARGETS})")"
 fi
 
 if [[ "${FUZZ_COVERAGE}" == "true" ]]; then
@@ -32,7 +35,7 @@ fi
 
 bazel coverage ${BAZEL_BUILD_OPTIONS} --test_output=all ${COVERAGE_TARGETS}
 
-COVERAGE_DIR="${SRCDIR}"/generated/coverage
+COVERAGE_DIR="${SRCDIR}"/generated/coverage && [[ ${FUZZ_COVERAGE} == "true" ]] && COVERAGE_DIR="${SRCDIR}"/generated/fuzz_coverage
 
 rm -rf "${COVERAGE_DIR}"
 mkdir -p "${COVERAGE_DIR}"
@@ -43,7 +46,12 @@ cp bazel-out/_coverage/_coverage_report.dat "${COVERAGE_DATA}"
 COVERAGE_VALUE=$(genhtml --prefix ${PWD} --output "${COVERAGE_DIR}" "${COVERAGE_DATA}" | tee /dev/stderr | grep lines... | cut -d ' ' -f 4)
 COVERAGE_VALUE=${COVERAGE_VALUE%?}
 
-[[ -z "${ENVOY_COVERAGE_DIR}" ]] || rsync -av "${COVERAGE_DIR}"/ "${ENVOY_COVERAGE_DIR}"
+if [ "$FUZZ_COVERAGE" == "true" ]
+then
+  [[ -z "${ENVOY_FUZZ_COVERAGE_DIR}" ]] || rsync -av "${COVERAGE_DIR}"/ "${ENVOY_FUZZ_COVERAGE_DIR}"
+else
+  [[ -z "${ENVOY_COVERAGE_DIR}" ]] || rsync -av "${COVERAGE_DIR}"/ "${ENVOY_COVERAGE_DIR}"
+fi
 
 if [[ "$VALIDATE_COVERAGE" == "true" ]]; then
   if [[ "${FUZZ_COVERAGE}" == "true" ]]; then