diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index 582d2e2216996..4ad2e9e9847c1 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -90,21 +90,28 @@ jobs: ciTarget: $(CI_TARGET) - job: coverage - displayName: "Linux-x64 coverage" - dependsOn: ["format"] + displayName: "Linux-x64" + dependsOn: ["release"] timeoutInMinutes: 360 pool: "x64-large" + strategy: + maxParallel: 2 + matrix: + coverage: + CI_TARGET: "coverage" + fuzz_coverage: + CI_TARGET: "fuzz_coverage" steps: - template: bazel.yml parameters: managedAgent: false - ciTarget: bazel.coverage + ciTarget: bazel.$(CI_TARGET) rbe: false # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces bazelBuildExtraOptions: "--test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base" - - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/coverage coverage' - displayName: "Upload Report to GCS" + - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/$(CI_TARGET) $(CI_TARGET)' + displayName: "Upload $(CI_TARGET) Report to GCS" env: ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) diff --git a/.bazelrc b/.bazelrc index bf32a37e6b026..d1a55923dc1b6 100644 --- a/.bazelrc +++ b/.bazelrc @@ -234,18 +234,20 @@ build:remote-ci --remote_cache=grpcs://remotebuildexecution.googleapis.com build:remote-ci --remote_executor=grpcs://remotebuildexecution.googleapis.com # Fuzz builds -build:asan-fuzzer --config=clang-asan +# -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION is passed in in the bazel build target +# rules for fuzz tests. Passing it in the CLI will cause dependencies to be build +# with the macro. Causing issues in RouteMatcherTest.TestRoutes that expect prod +# behavior from RE2 library. +build:asan-fuzzer --config=asan build:asan-fuzzer --define=FUZZING_ENGINE=libfuzzer build:asan-fuzzer --copt=-fsanitize=fuzzer-no-link build:asan-fuzzer --copt=-fno-omit-frame-pointer -build:asan-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION # Remove UBSAN halt_on_error to avoid crashing on protobuf errors. build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1 # Fuzzing without ASAN. This is useful for profiling fuzzers without any ASAN artifacts. build:plain-fuzzer --define=FUZZING_ENGINE=libfuzzer -build:plain-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -build:plain-fuzzer --copt=-fsanitize=fuzzer-no-link +build:plain-fuzzer --define ENVOY_CONFIG_ASAN=1 # Compile database generation config # We don't care about built binaries so always strip and use fastbuild. diff --git a/bazel/README.md b/bazel/README.md index a7e3f0165f247..62d7b6e8f2c0c 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -638,13 +638,19 @@ test/run_envoy_bazel_coverage.sh The summary results are printed to the standard output and the full coverage report is available in `generated/coverage/coverage.html`. +To generate coverage results for fuzz targets, use the `FUZZ_COVERAGE` environment variable, e.g.: +``` +FUZZ_COVERAGE=true VALIDATE_COVERAGE=false test/run_envoy_bazel_coverage.sh +``` +This generates a coverage report for fuzz targets after running the target for one minute against fuzzing engine libfuzzer using its coprus as initial seed inputs. The full coverage report will be available in `generated/fuzz_coverage/coverage.html`. + Coverage for every PR is available in Circle in the "artifacts" tab of the coverage job. You will need to navigate down and open "coverage.html" but then you can navigate per normal. NOTE: We have seen some issues with seeing the artifacts tab. If you can't see it, log out of Circle, and then log back in and it should start working. The latest coverage report for master is available -[here](https://storage.googleapis.com/envoy-postsubmit/master/coverage/index.html). +[here](https://storage.googleapis.com/envoy-postsubmit/master/coverage/index.html). The latest fuzz coverage report for master is available [here](https://storage.googleapis.com/envoy-postsubmit/master/fuzz_coverage/index.html). It's also possible to specialize the coverage build to a specified test or test dir. This is useful when doing things like exploring the coverage of a fuzzer over its corpus. This can be done by diff --git a/bazel/coverage/fuzz_coverage_wrapper.sh b/bazel/coverage/fuzz_coverage_wrapper.sh index 14c94bd545aea..0510befd60bc4 100755 --- a/bazel/coverage/fuzz_coverage_wrapper.sh +++ b/bazel/coverage/fuzz_coverage_wrapper.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -ex +set -x TEST_BINARY=$1 shift @@ -11,4 +11,7 @@ rm -rf fuzz_corpus mkdir -p fuzz_corpus/seed_corpus cp -r $@ fuzz_corpus/seed_corpus -${TEST_BINARY} fuzz_corpus -seed=${FUZZ_CORPUS_SEED:-1} -max_total_time=${FUZZ_CORPUS_TIME:-60} +# TODO(asraa): When fuzz targets are stable, remove error suppression and run coverage while fuzzing. +LLVM_PROFILE_FILE= ${TEST_BINARY} fuzz_corpus -seed=${FUZZ_CORPUS_SEED:-1} -max_total_time=${FUZZ_CORPUS_TIME:-60} -max_len=2048 || true + +${TEST_BINARY} fuzz_corpus -runs=0 diff --git a/ci/build_setup.sh b/ci/build_setup.sh index ee60c484ca4e1..97419a4256182 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -105,6 +105,9 @@ mkdir -p "${ENVOY_DELIVERY_DIR}" # This is where we copy the coverage report to. export ENVOY_COVERAGE_ARTIFACT="${ENVOY_BUILD_DIR}"/generated/coverage.tar.gz +# This is where we copy the fuzz coverage report to. +export ENVOY_FUZZ_COVERAGE_ARTIFACT="${ENVOY_BUILD_DIR}"/generated/fuzz_coverage.tar.gz + # This is where we dump failed test logs for CI collection. export ENVOY_FAILED_TEST_LOGS="${ENVOY_BUILD_DIR}"/generated/failed-testlogs mkdir -p "${ENVOY_FAILED_TEST_LOGS}" diff --git a/ci/do_ci.sh b/ci/do_ci.sh index d13c7be545bdd..0ab1da4204c7a 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -269,6 +269,13 @@ elif [[ "$CI_TARGET" == "bazel.coverage" ]]; then test/run_envoy_bazel_coverage.sh ${COVERAGE_TEST_TARGETS} collect_build_profile coverage exit 0 +elif [[ "$CI_TARGET" == "bazel.fuzz_coverage" ]]; then + setup_clang_toolchain + echo "bazel coverage build with fuzz tests ${COVERAGE_TEST_TARGETS}" + + FUZZ_COVERAGE=true test/run_envoy_bazel_coverage.sh ${COVERAGE_TEST_TARGETS} + collect_build_profile coverage + exit 0 elif [[ "$CI_TARGET" == "bazel.clang_tidy" ]]; then setup_clang_toolchain NUM_CPUS=$NUM_CPUS ci/run_clang_tidy.sh diff --git a/test/fuzz/README.md b/test/fuzz/README.md index aa3cf68057a3d..0104affcfd3d0 100644 --- a/test/fuzz/README.md +++ b/test/fuzz/README.md @@ -158,7 +158,15 @@ to provide fuzzers some interesting starting points for invalid inputs. ## Coverage reports Coverage reports, where individual lines are annotated with fuzzing hit counts, are a useful way to -understand the scope and efficacy of the Envoy fuzzers. You can generate such reports from the +understand the scope and efficacy of the Envoy fuzzers. You can generate fuzz coverage reports both locally, and using the OSS-Fuzz infrastructure. + +To generate fuzz coverage reports locally (see [Coverage builds](bazel/README.md), run +``` +FUZZ_COVERAGE=true test/run_envoy_bazel_coverage.sh +``` +This generates a coverage report after running the fuzz targets for one minute against the fuzzing engine libfuzzer and using the checked-in corpus as an initial seed. + +Otherwise, you can generate reports from the ClusterFuzz corpus following the general ClusterFuzz [instructions for profiling setup](https://github.com/google/oss-fuzz/blob/master/docs/code_coverage.md). diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 270ca9412ec71..7bf6cc92e36e2 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -19,12 +19,12 @@ if [[ $# -gt 0 ]]; then elif [[ -n "${COVERAGE_TARGET}" ]]; then COVERAGE_TARGETS=${COVERAGE_TARGET} else - # For fuzz builds, this overrides to just fuzz targets. - COVERAGE_TARGETS=//test/... && [[ ${FUZZ_COVERAGE} == "true" ]] && - COVERAGE_TARGETS="$(bazel query 'attr("tags", "fuzz_target", //test/...)')" + COVERAGE_TARGETS=//test/... fi if [[ "${FUZZ_COVERAGE}" == "true" ]]; then + # Filter targets to just fuzz tests. + COVERAGE_TARGETS=$(bazel query "attr("tags", "fuzz_target", ${COVERAGE_TARGETS})") BAZEL_BUILD_OPTIONS+=" --config=fuzz-coverage --test_tag_filters=-nocoverage" else BAZEL_BUILD_OPTIONS+=" --config=test-coverage --test_tag_filters=-nocoverage,-fuzz_target" @@ -36,7 +36,7 @@ bazel coverage ${BAZEL_BUILD_OPTIONS} ${COVERAGE_TARGETS} [[ -z "${ENVOY_BUILD_PROFILE}" ]] || cp -f "$(bazel info output_base)/command.profile.gz" "${ENVOY_BUILD_PROFILE}/coverage.profile.gz" || true [[ -z "${ENVOY_BUILD_DIR}" ]] || find bazel-testlogs/ -name test.log | tar zcf "${ENVOY_BUILD_DIR}/testlogs.tar.gz" -T - -COVERAGE_DIR="${SRCDIR}"/generated/coverage +COVERAGE_DIR="${SRCDIR}"/generated/coverage && [[ ${FUZZ_COVERAGE} == "true" ]] && COVERAGE_DIR="${SRCDIR}"/generated/fuzz_coverage rm -rf "${COVERAGE_DIR}" mkdir -p "${COVERAGE_DIR}" @@ -47,7 +47,12 @@ cp bazel-out/_coverage/_coverage_report.dat "${COVERAGE_DATA}" COVERAGE_VALUE=$(genhtml --prefix ${PWD} --output "${COVERAGE_DIR}" "${COVERAGE_DATA}" | tee /dev/stderr | grep lines... | cut -d ' ' -f 4) COVERAGE_VALUE=${COVERAGE_VALUE%?} -[[ -z "${ENVOY_COVERAGE_ARTIFACT}" ]] || tar zcf "${ENVOY_COVERAGE_ARTIFACT}" -C ${COVERAGE_DIR} --transform 's/^\./coverage/' . +if [ "${FUZZ_COVERAGE}" == "true" ] +then + [[ -z "${ENVOY_FUZZ_COVERAGE_ARTIFACT}" ]] || tar zcf "${ENVOY_FUZZ_COVERAGE_ARTIFACT}" -C ${COVERAGE_DIR} --transform 's/^\./fuzz_coverage/' . +else + [[ -z "${ENVOY_COVERAGE_ARTIFACT}" ]] || tar zcf "${ENVOY_COVERAGE_ARTIFACT}" -C ${COVERAGE_DIR} --transform 's/^\./coverage/' . +fi if [[ "$VALIDATE_COVERAGE" == "true" ]]; then if [[ "${FUZZ_COVERAGE}" == "true" ]]; then @@ -66,7 +71,7 @@ fi # We want to allow per_file_coverage to fail without exiting this script. set +e -if [[ "$VALIDATE_COVERAGE" == "true" ]]; then +if [[ "$VALIDATE_COVERAGE" == "true" ]] && [[ "{FUZZ_COVERAGE}" == "false" ]]; then echo "Checking per-extension coverage" output=$(./test/per_file_coverage.sh) diff --git a/test/server/BUILD b/test/server/BUILD index 9868dc06062ec..44ca9caa9500b 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -322,7 +322,6 @@ envoy_cc_fuzz_test( "//test/integration:integration_lib", "//test/mocks/server:options_mocks", "//test/mocks/server:hot_restart_mocks", - "//test/mocks/stats:stats_mocks", "//test/test_common:environment_lib", ] + select({ "//bazel:windows_x86_64": envoy_all_extensions(WINDOWS_SKIP_TARGETS), diff --git a/test/server/server_corpus/api_boost_crash b/test/server/server_corpus/api_boost_crash new file mode 100644 index 0000000000000..2dc13ace4237d --- /dev/null +++ b/test/server/server_corpus/api_boost_crash @@ -0,0 +1,138 @@ +node { + client_features: "&" +} +static_resources { + listeners { + name: " " + address { + pipe { + path: "aa\000" + } + } + transparent { + } + } +} +stats_sinks { + typed_config { + [type.googleapis.com/envoy.api.v2.route.Route] { + route { + retry_policy { + retriable_status_codes: 115 + retriable_status_codes: 116 + retriable_status_codes: 97 + retriable_status_codes: 116 + retriable_status_codes: 101 + retriable_status_codes: 113 + retriable_status_codes: 32 + retriable_status_codes: 123 + retriable_status_codes: 40 + retriable_status_codes: 36 + retriable_status_codes: 32 + retriable_status_codes: 42 + retriable_status_codes: 115 + retriable_status_codes: 116 + retriable_status_codes: 97 + retriable_status_codes: 116 + retriable_status_codes: 101 + retriable_status_codes: 113 + retriable_status_codes: 32 + retriable_status_codes: 123 + retriable_status_codes: 40 + retriable_status_codes: 36 + retriable_status_codes: 32 + retriable_status_codes: 32 + retriable_status_codes: 99 + retriable_status_codes: 108 + retriable_status_codes: 117 + retriable_status_codes: 115 + retriable_status_codes: 116 + retriable_status_codes: 101 + retriable_status_codes: 114 + retriable_status_codes: 123 + retriable_status_codes: 115 + } + } + } + } +} +stats_sinks { + typed_config { + [type.googleapis.com/envoy.api.v2.route.Route] { + route { + retry_policy { + retriable_status_codes: 115 + retriable_status_codes: 116 + retriable_status_codes: 97 + retriable_status_codes: 116 + retriable_status_codes: 105 + retriable_status_codes: 99 + retriable_status_codes: 95 + retriable_status_codes: 114 + retriable_status_codes: 101 + retriable_status_codes: 115 + retriable_status_codes: 111 + retriable_status_codes: 117 + retriable_status_codes: 114 + retriable_status_codes: 99 + retriable_status_codes: 65 + retriable_status_codes: 101 + retriable_status_codes: 32 + retriable_status_codes: 99 + retriable_status_codes: 115 + retriable_status_codes: 116 + retriable_status_codes: 97 + retriable_status_codes: 32 + retriable_status_codes: 32 + retriable_status_codes: 99 + retriable_status_codes: 115 + retriable_status_codes: 116 + retriable_status_codes: 97 + } + } + } + } +} +stats_sinks { + typed_config { + [type.googleapis.com/envoy.api.v2.route.Route] { + route { + retry_policy { + retriable_status_codes: 115 + retriable_status_codes: 116 + retriable_status_codes: 97 + retriable_status_codes: 116 + retriable_status_codes: 101 + retriable_status_codes: 113 + retriable_status_codes: 32 + retriable_status_codes: 123 + retriable_status_codes: 40 + retriable_status_codes: 36 + retriable_status_codes: 32 + retriable_status_codes: 42 + retriable_status_codes: 115 + retriable_status_codes: 116 + retriable_status_codes: 97 + retriable_status_codes: 116 + retriable_status_codes: 101 + retriable_status_codes: 113 + retriable_status_codes: 32 + retriable_status_codes: 123 + retriable_status_codes: 40 + retriable_status_codes: 36 + retriable_status_codes: 32 + retriable_status_codes: 32 + retriable_status_codes: 99 + retriable_status_codes: 108 + retriable_status_codes: 117 + retriable_status_codes: 115 + retriable_status_codes: 116 + retriable_status_codes: 101 + retriable_status_codes: 114 + retriable_status_codes: 123 + retriable_status_codes: 115 + } + } + } + } +} \ No newline at end of file