Code coverage integration in SRF

CMakeLists.txt

@@ -31,8 +31,9 @@ option(SRF_USE_CCACHE "Enable caching compilation results with ccache" OFF)
 option(SRF_USE_CLANG_TIDY "Enable running clang-tidy as part of the build process" OFF)
 option(SRF_USE_CONDA "Enables finding dependencies via conda instead of vcpkg. Note: This will disable vcpkg. All dependencies must be installed first in the conda environment" ON)
 option(SRF_USE_IWYU "Enable running include-what-you-use as part of the build process" OFF)
+option(SRF_WITH_CODECOV "Enable gcov code coverage" OFF)
 
-set(SRF_RAPIDS_VERSION "22.06" CACHE STRING "Which version of RAPIDs to build for. Sets default versions for RAPIDs CMake and RMM.")
+set(SRF_RAPIDS_VERSION "22.06" CACHE STRING "Which version of RAPIDS to build for. Sets default versions for RAPIDS CMake and RMM.")
 
 set(SRF_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
 mark_as_advanced(SRF_CACHE_DIR)
@@ -61,6 +62,7 @@ project(srf
   LANGUAGES C CXX
 )
 
+
 # Delay enabling CUDA until after we have determined our CXX compiler
 if(NOT DEFINED CMAKE_CUDA_HOST_COMPILER)
   message(STATUS "Setting CUDA host compiler to match CXX compiler: ${CMAKE_CXX_COMPILER}")
@@ -139,6 +141,44 @@ include(cmake/setup_compiler.cmake)
 # Setup IWYU if enabled
 include(cmake/setup_iwyu.cmake)
 
+# Include coverage tools if enabled
+if(SRF_WITH_CODECOV)
+  include(cmake/LibCodeCoverage.cmake)
+
+  setup_target_for_coverage_lcov(
+      NAME lcov-report
+      EXCLUDE
+        ".cache/*"
+        "tests/*"
+        "python/srf/tests/*"
+        "python/srf/_pysrf/tests/*"
+  )
+
+  setup_target_for_coverage_gcovr_xml(
+      NAME gcovr-xml-report
+      EXCLUDE
+        ".cache/*"
+        "tests/*"
+        "python/srf/tests/*"
+        "python/srf/_pysrf/tests/*"
+  )
+
+  setup_target_for_coverage_gcovr_html(
+      NAME gcovr-html-report
+      EXCLUDE
+        ".cache/*"
+        "tests/*"
+        "python/srf/tests/*"
+        "python/srf/_pysrf/tests/*"
+  )
+
+  # Anaconda doesn't seem to have a fastcov package -- disable for now
+  # setup_target_for_coverage_fastcov(
+  #     NAME fastcov-report
+  #     EXCLUDE ".cache/*"
+  # )
+endif()
+
 ####################################
 # - Begin SRF Targets --------------
 
@@ -280,6 +320,10 @@ set_target_properties(libsrf PROPERTIES OUTPUT_NAME ${PROJECT_NAME})
 # Finally, set the install RPATH to include the stubs folder for CUDA::nvml. If thats made private, this can be removed
 set_target_properties(libsrf PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib:\$ORIGIN:${CMAKE_INSTALL_PREFIX}/lib/stubs")
 
+if(SRF_WITH_CODECOV)
+  append_coverage_compiler_flags_to_target(libsrf)
+endif()
+
 ###################################################################################################
 # - install targets -------------------------------------------------------------------------------
 rapids_cmake_install_lib_dir(lib_dir)
@@ -318,6 +362,7 @@ endif ()
 
 add_subdirectory(tools/nvrpc)
 
+
 ###################################################################################################
 # - install export --------------------------------------------------------------------------------
 

ci/Jenkinsfile

@@ -38,7 +38,7 @@ pipeline {
         stage('Builds') {
             failFast true
             parallel {
-                stage('Build:linux:x86_64:gcc') {
+                stage('Build:linux:x86_64:gcc:release') {
                     options {
                         timeout(time: 1, unit: 'HOURS')
                     }
@@ -69,6 +69,38 @@ pipeline {
                         }
                     }
                 }
+                stage('Build:linux:x86_64:gcc:debug') {
+                    options {
+                        timeout(time: 1, unit: 'HOURS')
+                    }
+                    environment {
+                        BUILD_CC= "gcc-coverage"
+                        PARALLEL_LEVEL = '16'
+                        HOME = "${WORKSPACE}"
+                    }
+                    agent {
+                        docker {
+                            image 'gpuci/rapidsai-driver:21.10-cuda11.4-devel-ubuntu20.04-py3.8'
+                            label 'cpu4'
+                        }
+                    }
+                    steps {
+                        cleanWs(
+                            deleteDirs: true,
+                            externalDelete: 'sudo rm -rf %s'
+                        )
+                        checkout scm
+                        withCredentials([[
+                            $class: 'AmazonWebServicesCredentialsBinding',
+                            credentialsId: "aws-s3-gpuci",
+                            accessKeyVariable: 'AWS_ACCESS_KEY_ID',
+                            secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'
+                        ]])
+                        {
+                            sh "${WORKSPACE}/ci/scripts/jenkins/build.sh"
+                        }
+                    }
+                }
                 stage('Build:linux:x86_64:clang') {
                     options {
                         timeout(time: 1, unit: 'HOURS')
@@ -132,13 +164,44 @@ pipeline {
                         }
                     }
                 }
-
             }
         }
         stage('Tests') {
             failFast true
             parallel {
-                stage('Test') {
+                stage('TestDebug') {
+                     options {
+                         timeout(time: 1, unit: 'HOURS')
+                     }
+                     agent {
+                         docker {
+                             image 'gpuci/rapidsai:21.10-cuda11.4-devel-ubuntu20.04-py3.8'
+                             label 'driver-495'
+                             args '--cap-add=sys_nice'
+                         }
+                     }
+                     environment {
+                         BUILD_TYPE = "Debug"
+                         HOME = "${WORKSPACE}"
+                     }
+                     steps {
+                         cleanWs(
+                             deleteDirs: true,
+                             externalDelete: 'sudo rm -rf %s'
+                         )
+                         checkout scm
+                         withCredentials([[
+                             $class: 'AmazonWebServicesCredentialsBinding',
+                             credentialsId: "aws-s3-gpuci",
+                             accessKeyVariable: 'AWS_ACCESS_KEY_ID',
+                             secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'
+                         ]])
+                         {
+                             sh "${WORKSPACE}/ci/scripts/jenkins/test.sh"
+                         }
+                     }
+                }
+                stage('TestRelease') {
                     options {
                         timeout(time: 1, unit: 'HOURS')
                     }
@@ -150,6 +213,7 @@ pipeline {
                         }
                     }
                     environment {
+                        BUILD_TYPE = "Release"
                         HOME = "${WORKSPACE}"
                     }
                     steps {

ci/conda/environments/dev_env.yml

@@ -14,6 +14,7 @@ dependencies:
   - doxygen=1.9.2
   - flatbuffers=2.0
   - gcc_linux-64=9.4
+  - gcovr=5.1
   - gflags=2.2
   - git>=2.35.3 # Needed for wildcards on safe.directory
   - glog=0.6
@@ -22,6 +23,7 @@ dependencies:
   - grpc-cpp=1.45
   - gtest=1.10
   - gxx_linux-64=9.4
+  - lcov=1.15
   - libhwloc=2.5
   - librmm=22.06
   - libtool

ci/scripts/copyright.py

@@ -33,7 +33,7 @@
     re.compile(r"[.]flake8[.]cython$"),
     re.compile(r"meta[.]yaml$")
 ]
-ExemptFiles = ['.cache', 'build', 'cmake/LibFindMacros.cmake', 'versioneer.py']
+ExemptFiles = ['.cache', 'build', 'cmake/LibCodeCoverage.cmake', 'cmake/LibFindMacros.cmake', 'versioneer.py']
 
 # this will break starting at year 10000, which is probably OK :)
 CheckSimple = re.compile(r"Copyright *(?:\(c\))? *(\d{4}),? *NVIDIA C(?:ORPORATION|orporation)")

ci/scripts/jenkins/build.sh

@@ -39,6 +39,11 @@ if [[ "${BUILD_CC}" == "gcc" ]]; then
     gcc --version
     g++ --version
     CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES} ${CMAKE_CACHE_FLAGS}"
+elif [[ "${BUILD_CC}" == "gcc-coverage" ]]; then
+    gpuci_logger "Building with GCC with gcov profile '-g -fprofile-arcs -ftest-coverage"
+    gcc --version
+    g++ --version
+    CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES} ${CMAKE_BUILD_WITH_CODECOV} ${CMAKE_CACHE_FLAGS}"
 else
     gpuci_logger "Installing Clang"
     mamba env update -q -n srf --file ${SRF_ROOT}/ci/conda/environments/clang_env.yml

ci/scripts/jenkins/common.sh

@@ -31,6 +31,7 @@ export NUM_PROC=${PARALLEL_LEVEL:-$(nproc)}
 export CONDA_ENV_YML="${SRF_ROOT}/ci/conda/environments/dev_env.yml"
 
 export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DSRF_BUILD_BENCHMARKS=ON -DSRF_BUILD_EXAMPLES=ON -DSRF_BUILD_PYTHON=ON -DSRF_BUILD_TESTS=ON -DSRF_USE_CONDA=ON"
+export CMAKE_BUILD_WITH_CODECOV="-DSRF_WITH_CODECOV=ON"
 
 # Set the depth to allow git describe to work
 export GIT_DEPTH=1000

ci/scripts/jenkins/test.sh

@@ -59,13 +59,22 @@ pytest -v --junit-xml=${WORKSPACE_TMP}/report_pytest.xml
 PYTEST_RESULTS=$?
 set -e
 
+if [[ "${BUILD_TYPE}" == "Debug" ]]; then
+  gpuci_logger "Generating codecov report"
+  cd ${SRF_ROOT}
+  cmake --build build --target gcovr-html-report
+
+  gpuci_logger "Archiving codecov report"
+  tar cfj ${WORKSPACE_TMP}/coverage_reports.tar.bz ${SRF_ROOT}/build/gcovr-html-report
+  aws s3 cp ${WORKSPACE_TMP}/coverage_reports.tar.bz "${ARTIFACT_URL}/coverage_reports.tar.bz"
+fi
+
 gpuci_logger "Archiving test reports"
 cd $(dirname ${REPORTS_DIR})
 tar cfj ${WORKSPACE_TMP}/test_reports.tar.bz $(basename ${REPORTS_DIR})
 
 gpuci_logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
 aws s3 cp ${WORKSPACE_TMP}/test_reports.tar.bz "${ARTIFACT_URL}/test_reports.tar.bz"
 
-
 TEST_RESULTS=$(($CTEST_RESULTS+$PYTEST_RESULTS))
 exit ${TEST_RESULTS}