coverage: use LLVM and bazel native coverage

ci/do_ci.sh

@@ -241,22 +241,11 @@ elif [[ "$CI_TARGET" == "bazel.api" ]]; then
     @envoy_api//tools:tap2pcap_test
   exit 0
 elif [[ "$CI_TARGET" == "bazel.coverage" ]]; then
-  setup_gcc_toolchain
+  setup_clang_toolchain
   echo "bazel coverage build with tests ${TEST_TARGETS}"
 
-  # gcovr is a pain to run with `bazel run`, so package it up into a
-  # relocatable and hermetic-ish .par file.
-  bazel build --python_version=PY2 @com_github_gcovr_gcovr//:gcovr.par
-  export GCOVR="/tmp/gcovr.par"
-  cp -f "${ENVOY_SRCDIR}/bazel-bin/external/com_github_gcovr_gcovr/gcovr.par" ${GCOVR}
-
-  # Reduce the amount of memory and number of cores Bazel tries to use to
-  # prevent it from launching too many subprocesses. This should prevent the
-  # system from running out of memory and killing tasks. See discussion on
-  # https://github.com/envoyproxy/envoy/pull/5611.
-  # TODO(akonradi): use --local_cpu_resources flag once Bazel has a release
-  # after 0.21.
-  [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --local_resources=12288,4,1"
+  # LLVM coverage is a memory hog too.
+  [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --local_cpu_resources=6"
 
   test/run_envoy_bazel_coverage.sh ${TEST_TARGETS}
   collect_build_profile coverage

test/run_envoy_bazel_coverage.sh

@@ -3,21 +3,11 @@
 set -e
 
 [[ -z "${SRCDIR}" ]] && SRCDIR="${PWD}"
-[[ -z "${GCOVR_DIR}" ]] && GCOVR_DIR="${SRCDIR}/bazel-$(basename "${SRCDIR}")"
-[[ -z "${TESTLOGS_DIR}" ]] && TESTLOGS_DIR="${SRCDIR}/bazel-testlogs"
-[[ -z "${BAZEL_COVERAGE}" ]] && BAZEL_COVERAGE=bazel
-[[ -z "${GCOVR}" ]] && GCOVR=gcovr
-[[ -z "${WORKSPACE}" ]] && WORKSPACE=envoy
 [[ -z "${VALIDATE_COVERAGE}" ]] && VALIDATE_COVERAGE=true
 
 echo "Starting run_envoy_bazel_coverage.sh..."
 echo "    PWD=$(pwd)"
 echo "    SRCDIR=${SRCDIR}"
-echo "    GCOVR_DIR=${GCOVR_DIR}"
-echo "    TESTLOGS_DIR=${TESTLOGS_DIR}"
-echo "    BAZEL_COVERAGE=${BAZEL_COVERAGE}"
-echo "    GCOVR=${GCOVR}"
-echo "    WORKSPACE=${WORKSPACE}"
 echo "    VALIDATE_COVERAGE=${VALIDATE_COVERAGE}"
 
 # This is the target that will be run to generate coverage data. It can be overridden by consumer
@@ -31,97 +21,43 @@ else
   COVERAGE_TARGETS=//test/...
 fi
 
-# This is where we are going to copy the .gcno files into.
-GCNO_ROOT=bazel-out/k8-dbg/bin/test/coverage/coverage_tests.runfiles/"${WORKSPACE}"
-echo "    GCNO_ROOT=${GCNO_ROOT}"
-
-echo "Cleaning .gcno from previous coverage runs..."
-NUM_PREVIOUS_GCNO_FILES=0
-for f in $(find -L "${GCNO_ROOT}" -name "*.gcno")
-do
-  rm -f "${f}"
-  let NUM_PREVIOUS_GCNO_FILES=NUM_PREVIOUS_GCNO_FILES+1
-done
-echo "Cleanup completed. ${NUM_PREVIOUS_GCNO_FILES} files deleted."
-
 # Make sure //test/coverage:coverage_tests is up-to-date.
 SCRIPT_DIR="$(realpath "$(dirname "$0")")"
-(BAZEL_BIN="${BAZEL_COVERAGE}" "${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS})
-
-echo "Cleaning .gcda/.gcov from previous coverage runs..."
-NUM_PREVIOUS_GCOV_FILES=0
-for f in $(find -L "${GCOVR_DIR}" -name "*.gcda" -o -name "*.gcov")
-do
-  rm -f "${f}"
-  let NUM_PREVIOUS_GCOV_FILES=NUM_PREVIOUS_GCOV_FILES+1
-done
-echo "Cleanup completed. ${NUM_PREVIOUS_GCOV_FILES} files deleted."
+NO_GCOV=1 "${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS}
 
-# Force dbg for path consistency later, don't include debug code in coverage.
-BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} -c dbg --copt=-DNDEBUG"
-
-# Run all tests under "bazel test", no sandbox. We're going to generate the
-# .gcda inplace in the bazel-out/ directory. This is in contrast to the "bazel
-# coverage" method, which is currently broken for C++ (see
-# https://github.com/bazelbuild/bazel/issues/1118). This works today as we have
-# a single coverage test binary and do not require the "bazel coverage" support
-# for collecting multiple traces and glueing them together.
-"${BAZEL_COVERAGE}" test //test/coverage:coverage_tests ${BAZEL_BUILD_OPTIONS} \
-  --cache_test_results=no --cxxopt="--coverage" --cxxopt="-DENVOY_CONFIG_COVERAGE=1" \
-  --linkopt="--coverage" --define ENVOY_CONFIG_COVERAGE=1 --test_output=streamed \
-  --strategy=Genrule=standalone --spawn_strategy=standalone --test_timeout=4000 \
-  --test_arg="--log-path /dev/null" --test_arg="-l trace"
-
-# The Bazel build has a lot of whack in it, in particular generated files, headers from external
-# deps, etc. So, we exclude this from gcov to avoid false reporting of these files in the html and
-# stats. The #foo# pattern is because gcov produces files such as
-# bazel-out#local-fastbuild#bin#external#spdlog_git#_virtual_includes#spdlog#spdlog#details#pattern_formatter_impl.h.gcov.
-# To find these while modifying this regex, perform a gcov run with -k set.
-[[ -z "${GCOVR_EXCLUDE_REGEX}" ]] && GCOVR_EXCLUDE_REGEX=".*pb\\..*|test#.*|.*#test#.*|external#.*|.*#external#.*|.*#prebuilt#.*|.*#config_validation#.*|.*#chromium_url#.*"
-[[ -z "${GCOVR_EXCLUDE_DIR}" ]] && GCOVR_EXCLUDE_DIR=".*/external/.*"
+BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD_OPTIONS} \
+    -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... \
+    --test_timeout=2000 --cxxopt="-DENVOY_CONFIG_COVERAGE=1" --test_output=streamed \
+    --test_arg="--log-path /dev/null" --test_arg="-l trace" --test_env=HEAPCHECK= \
+    --test_filter='-QuicPlatformTest.QuicStackTraceTest' //test/coverage:coverage_tests
 
 COVERAGE_DIR="${SRCDIR}"/generated/coverage
 mkdir -p "${COVERAGE_DIR}"
-COVERAGE_SUMMARY="${COVERAGE_DIR}/coverage_summary.txt"
-
-# Copy .gcno objects into the same location that we find the .gcda.
-# TODO(htuch): Should use rsync, but there are some symlink loops to fight.
-echo "Finding and copying .gcno files in GCOVR_DIR: ${GCOVR_DIR}"
-mkdir -p ${GCNO_ROOT}
-NUM_GCNO_FILES=0
-for f in $(find -L bazel-out/ -name "*.gcno")
-do
-  cp --parents "$f" ${GCNO_ROOT}/
-  let NUM_GCNO_FILES=NUM_GCNO_FILES+1
-done
-echo "OK: copied ${NUM_GCNO_FILES} .gcno files"
 
-# gcovr is extremely picky about where it is run and where the paths of the
-# original source are relative to its execution location.
-cd -P "${GCOVR_DIR}"
-echo "Running gcovr in $(pwd)..."
-time "${GCOVR}" -v --gcov-exclude="${GCOVR_EXCLUDE_REGEX}" \
-  --exclude-directories="${GCOVR_EXCLUDE_DIR}" -r . \
-  --html --html-details --exclude-unreachable-branches --print-summary \
-  -o "${COVERAGE_DIR}"/coverage.html > "${COVERAGE_SUMMARY}"
+COVERAGE_IGNORE_REGEX="(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/|/test/|/tmp)"
+COVERAGE_BINARY="bazel-bin/test/coverage/coverage_tests"
+COVERAGE_DATA="bazel-out/k8-fastbuild/testlogs/test/coverage/coverage_tests/coverage.dat"
 
-# Clean up the generated test/coverage/BUILD file: subsequent bazel invocations
-# can choke on it if it references things that changed since the last coverage
-# run.
-rm "${SRCDIR}"/test/coverage/BUILD
+echo "Generating report..."
+llvm-cov show "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" -Xdemangler=c++filt \
+  -ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" -output-dir=${COVERAGE_DIR} -format=html
+sed -i -e 's|>proc/self/cwd/|>|g' "${COVERAGE_DIR}/index.html"
+sed -i -e 's|>bazel-out/[^/]*/bin/\([^/]*\)/[^<]*/_virtual_includes/[^/]*|>\1|g' "${COVERAGE_DIR}/index.html"
 
 [[ -z "${ENVOY_COVERAGE_DIR}" ]] || rsync -av "${COVERAGE_DIR}"/ "${ENVOY_COVERAGE_DIR}"
 
 if [ "$VALIDATE_COVERAGE" == "true" ]
 then
-  COVERAGE_VALUE=$(grep -Po 'lines: \K(\d|\.)*' "${COVERAGE_SUMMARY}")
-  COVERAGE_THRESHOLD=97.5
+  COVERAGE_VALUE=$(llvm-cov export "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" \
+    -ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" -summary-only | \
+    python3 -c "import sys, json; print(json.load(sys.stdin)['data'][0]['totals']['lines']['percent'])")
+  COVERAGE_THRESHOLD=97.0
   COVERAGE_FAILED=$(echo "${COVERAGE_VALUE}<${COVERAGE_THRESHOLD}" | bc)
   if test ${COVERAGE_FAILED} -eq 1; then
       echo Code coverage ${COVERAGE_VALUE} is lower than limit of ${COVERAGE_THRESHOLD}
       exit 1
   else
       echo Code coverage ${COVERAGE_VALUE} is good and higher than limit of ${COVERAGE_THRESHOLD}
   fi
-  echo "HTML coverage report is in ${COVERAGE_DIR}/coverage.html"
 fi
+echo "HTML coverage report is in ${COVERAGE_DIR}/index.html"