From c8569c916ea28670e296bb3f0069005f58077e0c Mon Sep 17 00:00:00 2001 From: apcraig Date: Wed, 26 May 2021 15:31:26 -0600 Subject: [PATCH 1/2] Update the coverage implementation. The gcov/lcov tool occasionally generates false negatives or incorrect statistics with respect to coverage for Fortran 90 continuation lines (using "&"). In some cases, within a single continuation block, some lines have hits which are correctly counted while other lines have misses which are incorrect and also counted. This seems to be partly associate with the lcov -a feature that we use to aggregate multiple tests, but it also probably is ultimately created by gcov because it doesn't seem to handle skipped vs missed lines entirely consistently. With this modification, --coverage creates a temporary sandbox where the tests are run. Prior to running the tests, lcov_modify_source.sh is invoked. " ! LCOV_EXCL_LINE" (a special defined string in lcov) is added to the end of some source code lines to request that gcov exclude them. The lines excluded are Fortran 90 continuation lines. In all cases the first and last line of a continuation block and lines that continue after an "if" are never excluded. This seems to improve the accuracy of the coverage output overall. Several other attempts were made to modify the lcov and geninfo perl scripts to handle the Fortran90 continuation line filtering there without success, so this solution was implemented. --- cice.setup | 12 ++++++ cicecore/cicedynB/dynamics/ice_dyn_evp.F90 | 3 +- .../scripts/tests/lcov_modify_source.sh | 42 +++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100755 configuration/scripts/tests/lcov_modify_source.sh diff --git a/cice.setup b/cice.setup index 8511f4cae..8dc46005a 100755 --- a/cice.setup +++ b/cice.setup @@ -390,6 +390,18 @@ if ((${dosuite} == 1 || ${dotest} == 1) && ${testid} == ${spval}) then exit -1 endif +# This creates a new sandbox and modifies the source code for "improved" lcov analysis +# Turn this if block off if you don't want coverage to do that +if ($coverage == 1) then + set sandbox_lcov = ${ICE_SANDBOX}/../cice_lcov_${sdate}-${stime} + cp -p -r ${ICE_SANDBOX} ${sandbox_lcov} + echo "shifting to sandbox = ${sandbox_lcov}" + set ICE_SANDBOX = ${sandbox_lcov} + set ICE_SCRIPTS = "${ICE_SANDBOX}/configuration/scripts" + cd ${ICE_SANDBOX} + ${ICE_SCRIPTS}/tests/lcov_modify_source.sh +endif + #--------------------------------------------------------------------- # Setup tsfile and test suite support stuff diff --git a/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 b/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 index d8ce42681..2206e0de7 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 @@ -361,8 +361,7 @@ subroutine evp (dt) first_time = .false. endif if (trim(grid_type) == 'tripole') then - call abort_ice(trim(subname)//' & - & Kernel not tested on tripole grid. Set kevp_kernel=0') + call abort_ice(trim(subname)//' Kernel not tested on tripole grid. Set kevp_kernel=0') endif call ice_dyn_evp_1d_copyin( & nx_block,ny_block,nblocks,nx_global+2*nghost,ny_global+2*nghost, & diff --git a/configuration/scripts/tests/lcov_modify_source.sh b/configuration/scripts/tests/lcov_modify_source.sh new file mode 100755 index 000000000..5d1113c5f --- /dev/null +++ b/configuration/scripts/tests/lcov_modify_source.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +filelist=`find cicecore icepack -type f -name "*.F90"` +LCOV_EXCL=" ! LCOV_EXCL_LINE" + +#echo $filelist + +for file in $filelist; do + + echo $file + ofile=${file}.orig + nfile=${file} + + mv ${file} ${file}.orig + + # line by line making sure each line has a trailing newline (-n) + # preserve whitespace (IFS) + # and include backslashes (-r) + IFS='' + contblock=0 + cat $ofile | while read -r line || [[ -n $oline ]]; do + + if [[ $contblock == 1 ]]; then + if [[ $line =~ ^.*"&".*$ ]]; then +# echo ${line} ${LCOV_EXCL} + echo ${line} ${LCOV_EXCL} >> ${nfile} + else + contblock=0 +# echo ${line} + echo ${line} >> ${nfile} + fi + else + echo ${line} >> ${nfile} + if [[ $line =~ ^\s*.*"&".*$ && ! $line =~ ^\s*( if ).*$ ]]; then + contblock=1 +# echo ${line} + fi + fi + + done + +done From c13c2732d7d5ab30278c0e2d51447924a760fb87 Mon Sep 17 00:00:00 2001 From: apcraig Date: Wed, 26 May 2021 16:16:06 -0600 Subject: [PATCH 2/2] update documentation --- configuration/scripts/tests/lcov_modify_source.sh | 10 ++++++---- doc/source/user_guide/ug_testing.rst | 5 ++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/configuration/scripts/tests/lcov_modify_source.sh b/configuration/scripts/tests/lcov_modify_source.sh index 5d1113c5f..ceadca4f4 100755 --- a/configuration/scripts/tests/lcov_modify_source.sh +++ b/configuration/scripts/tests/lcov_modify_source.sh @@ -18,22 +18,24 @@ for file in $filelist; do # and include backslashes (-r) IFS='' contblock=0 - cat $ofile | while read -r line || [[ -n $oline ]]; do + cat $ofile | while read -r line || [[ -n $line ]]; do if [[ $contblock == 1 ]]; then + # in a continuation block if [[ $line =~ ^.*"&".*$ ]]; then -# echo ${line} ${LCOV_EXCL} + # found another continuation line, add exclude string and write out line echo ${line} ${LCOV_EXCL} >> ${nfile} else + # continuation block ends, write out line contblock=0 -# echo ${line} echo ${line} >> ${nfile} fi else + # not in a continuation block, write out line echo ${line} >> ${nfile} if [[ $line =~ ^\s*.*"&".*$ && ! $line =~ ^\s*( if ).*$ ]]; then + # new continuation block found contblock=1 -# echo ${line} fi fi diff --git a/doc/source/user_guide/ug_testing.rst b/doc/source/user_guide/ug_testing.rst index 4a4ced555..5a289db6a 100644 --- a/doc/source/user_guide/ug_testing.rst +++ b/doc/source/user_guide/ug_testing.rst @@ -715,7 +715,10 @@ This argument turns on special compiler flags including reduced optimization and invokes the gcov tool. Once runs are complete, either lcov or codecov can be used to analyze the results. This option is currently only available with the gnu compiler and on a few systems -with modified Macros files. +with modified Macros files. In the current implementation, when ``--coverage`` is +invoked, the sandbox is copied to a new sandbox called something like cice_lcov_yymmdd-hhmmss. +The source code in the new sandbox is modified slightly to improve coverage statistics +and the full coverage suite is run there. At the present time, the ``--coverage`` flag invokes the lcov analysis automatically by running the **report_lcov.csh** script in the test suite directory. The output