From 4ba6f4d7e4b2e962a1b528734b791c25aa3a6174 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Fri, 1 Apr 2022 21:35:47 +0000 Subject: [PATCH 01/13] Add call to getgb2 to read new high-res global afwa data. Fixes #635. --- sorc/emcsfc_snow2mdl.fd/snowdat.F90 | 81 ++++++++++++++++------------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/sorc/emcsfc_snow2mdl.fd/snowdat.F90 b/sorc/emcsfc_snow2mdl.fd/snowdat.F90 index c9f91ee65..d0c5ec1de 100755 --- a/sorc/emcsfc_snow2mdl.fd/snowdat.F90 +++ b/sorc/emcsfc_snow2mdl.fd/snowdat.F90 @@ -529,14 +529,23 @@ end subroutine readnesdis !! !! @author George Gayno org: w/np2 @date 2005-Dec-16 subroutine readafwa + use grib_mod + implicit none - integer, parameter :: iunit=11 + integer, parameter :: iunit=17 integer :: jgds(200), jpds(200), kgds(200), kpds(200) integer :: istat integer :: lugi, lskip, numbytes, numpts, message_num integer :: isgrib + integer :: j, k, jdisc, jpdtn, jgdtn + integer :: jpdt(200), jgdt(200), jids(200) + + logical :: unpack + + type(gribfield) :: gfld + bad_afwa_nh=.false. bad_afwa_sh=.false. bad_afwa_global=.false. @@ -557,7 +566,7 @@ subroutine readafwa if ( len_trim(afwa_snow_global_file) > 0 ) then - print*,"- OPEN AND READ AFWA SNOW FILE ", trim(afwa_snow_global_file) + print*,"- OPEN AND READ global AFWA SNOW FILE ", trim(afwa_snow_global_file) call baopenr (iunit, afwa_snow_global_file, istat) if (istat /= 0) then print*,'- FATAL ERROR: BAD OPEN OF FILE, ISTAT IS ', istat @@ -565,48 +574,47 @@ subroutine readafwa call errexit(60) end if -!----------------------------------------------------------------------- -! tell degribber to look for requested data. -!----------------------------------------------------------------------- + call grib2_null(gfld) - lugi = 0 - lskip = -1 - jpds = -1 - jgds = -1 - jpds(5) = 66 ! snow depth - kpds = jpds - kgds = jgds + jdisc = 0 ! search for discipline; 0 - meteorological products + j = 0 ! search at beginning of file. + lugi = 0 ! no grib index file + jids = -9999 + jgdt = -9999 + jgdtn = -1 + jpdtn = 0 ! Search for product def template 0 - analysis or forecast + jpdt = -9999 ! array of values in product definition template 4.n + jpdt(1) = 1 ! parameter category - moisture Sec4 oct 10 + jpdt(2) = 11 ! parameter - snow depth Sec4 oct 11 + unpack = .true. ! unpack data - print*,"- GET GRIB HEADER" - call getgbh(iunit, lugi, lskip, jpds, jgds, numbytes, & - numpts, message_num, kpds, kgds, istat) + call getgb2(iunit, lugi, j, jdisc, jids, jpdtn, jpdt, jgdtn, jgdt, & + unpack, k, gfld, istat) - if (istat /= 0) then - print*,"- FATAL ERROR: BAD DEGRIB OF HEADER. ISTAT IS ", istat - call w3tage('SNOW2MDL') - call errexit(61) - end if + print*,'after getgb2 ',istat + if(istat /= 0) stop + + print*,'the data ',maxval(gfld%fld),minval(gfld%fld) + print*,'ipdtnum ', gfld%ipdtnum + print*,'ipdtmpl ', gfld%ipdtmpl + print*,'igdtnum ', gfld%igdtnum + print*,'igdtmpl ', gfld%igdtmpl + print*,'idsect ', gfld%idsect + + print*,"- DATA VALID AT (YYMMDDHH): ", gfld%idsect(6:9) + print*,"- DEGRIB SNOW DEPTH." - iafwa = kgds(2) - jafwa = kgds(3) - afwa_res = float(kgds(10))*0.001*111.0 ! in km. + call gdt_to_gds(gfld%igdtnum, gfld%igdtmpl, gfld%igdtlen, kgds_afwa_global, & + iafwa, jafwa, afwa_res) - print*,"- DATA VALID AT (YYMMDDHH): ", kpds(8:11) - print*,"- DEGRIB SNOW DEPTH." + print*,'afwares ',afwa_res + print*,'i/jafwa ',iafwa,jafwa allocate(bitmap_afwa_global(iafwa,jafwa)) allocate(snow_dep_afwa_global(iafwa,jafwa)) - call getgb(iunit, lugi, (iafwa*jafwa), lskip, jpds, jgds, & - numpts, lskip, kpds, kgds, bitmap_afwa_global, snow_dep_afwa_global, istat) - - if (istat /= 0) then - print*,"- FATAL ERROR: BAD DEGRIB OF DATA. ISTAT IS ", istat - call w3tage('SNOW2MDL') - call errexit(61) - end if - - kgds_afwa_global = kgds + snow_dep_afwa_global = reshape(gfld%fld, (/iafwa,jafwa/)) + bitmap_afwa_global = reshape(gfld%bmap, (/iafwa,jafwa/)) call baclose(iunit, istat) @@ -620,6 +628,8 @@ subroutine readafwa use_nh_afwa=.false. ! use global or hemispheric files. not both. use_sh_afwa=.false. + print*,'got here' + stop return ! use global or hemispheric files. not both. else @@ -1008,7 +1018,6 @@ subroutine nh_climo_check(kgds_data,snow_data,bitmap_data,idata,jdata,isrc,bad) print*,"- WARNING: PROBLEM READING GRIB FILE ", iret print*,"- WILL NOT PERFORM QC." deallocate(rlon_data,rlat_data) - deallocate(climo, bitmap_clim) call baclose(lugb,iret) return endif From 232690405a4bd6b2e59c5471dcd933115b47dfa5 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Mon, 4 Apr 2022 20:43:59 +0000 Subject: [PATCH 02/13] Cleanup read of afwa grib2 data. Fixes #635 --- sorc/emcsfc_snow2mdl.fd/snowdat.F90 | 66 +++++++++++++---------------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/sorc/emcsfc_snow2mdl.fd/snowdat.F90 b/sorc/emcsfc_snow2mdl.fd/snowdat.F90 index d0c5ec1de..1ef788f18 100755 --- a/sorc/emcsfc_snow2mdl.fd/snowdat.F90 +++ b/sorc/emcsfc_snow2mdl.fd/snowdat.F90 @@ -519,7 +519,7 @@ end subroutine readnesdis !! !! files: !! input: -!! - global afwa data in grib 1 (if selected) +!! - global afwa data in grib 2 (if selected) !! - nh afwa data in grib 1 (if selected) !! - sh afwa data in grib 1 (if selected) !! @@ -535,16 +535,14 @@ subroutine readafwa integer, parameter :: iunit=17 integer :: jgds(200), jpds(200), kgds(200), kpds(200) - integer :: istat + integer :: istat, isgrib integer :: lugi, lskip, numbytes, numpts, message_num - integer :: isgrib - integer :: j, k, jdisc, jpdtn, jgdtn integer :: jpdt(200), jgdt(200), jids(200) logical :: unpack - type(gribfield) :: gfld + type(gribfield) :: gfld bad_afwa_nh=.false. bad_afwa_sh=.false. @@ -564,6 +562,10 @@ subroutine readafwa return end if +!----------------------------------------------------------------------- +! If chosen, read global AFWA GRIB2 file. +!----------------------------------------------------------------------- + if ( len_trim(afwa_snow_global_file) > 0 ) then print*,"- OPEN AND READ global AFWA SNOW FILE ", trim(afwa_snow_global_file) @@ -574,42 +576,36 @@ subroutine readafwa call errexit(60) end if - call grib2_null(gfld) + call grib2_null(gfld) - jdisc = 0 ! search for discipline; 0 - meteorological products - j = 0 ! search at beginning of file. - lugi = 0 ! no grib index file - jids = -9999 - jgdt = -9999 - jgdtn = -1 - jpdtn = 0 ! Search for product def template 0 - analysis or forecast - jpdt = -9999 ! array of values in product definition template 4.n - jpdt(1) = 1 ! parameter category - moisture Sec4 oct 10 - jpdt(2) = 11 ! parameter - snow depth Sec4 oct 11 - unpack = .true. ! unpack data + jdisc = 0 ! Search for discipline; 0 - meteorological products + j = 0 ! Search at beginning of file. + lugi = 0 ! No grib index file. + jids = -9999 ! Identification section, set to wildcard. + jgdt = -9999 ! Grid definition template, set to wildcard. + jgdtn = -1 ! Grid definition template number, set to wildcard. + jpdtn = 0 ! Search for product definition template number 0 - analysis or forecast + jpdt = -9999 ! Product definition template (Sec 4), initialize to wildcard. + jpdt(1) = 1 ! Search for parameter category 1 (Sec 4 oct 10) - + ! moisture. + jpdt(2) = 11 ! Search for parameter 11 (Sec 4 oct 11) - snow depth. + unpack = .true. ! Unpack data. - call getgb2(iunit, lugi, j, jdisc, jids, jpdtn, jpdt, jgdtn, jgdt, & - unpack, k, gfld, istat) + call getgb2(iunit, lugi, j, jdisc, jids, jpdtn, jpdt, jgdtn, jgdt, & + unpack, k, gfld, istat) - print*,'after getgb2 ',istat - if(istat /= 0) stop + if (istat /= 0) then + print*,"- FATAL ERROR: BAD DEGRIB OF GLOBAL DATA. ISTAT IS ", istat + call w3tage('SNOW2MDL') + call errexit(61) + end if - print*,'the data ',maxval(gfld%fld),minval(gfld%fld) - print*,'ipdtnum ', gfld%ipdtnum - print*,'ipdtmpl ', gfld%ipdtmpl - print*,'igdtnum ', gfld%igdtnum - print*,'igdtmpl ', gfld%igdtmpl - print*,'idsect ', gfld%idsect - print*,"- DATA VALID AT (YYMMDDHH): ", gfld%idsect(6:9) print*,"- DEGRIB SNOW DEPTH." - call gdt_to_gds(gfld%igdtnum, gfld%igdtmpl, gfld%igdtlen, kgds_afwa_global, & + call gdt_to_gds(gfld%igdtnum, gfld%igdtmpl, gfld%igdtlen, kgds_afwa_global, & iafwa, jafwa, afwa_res) - print*,'afwares ',afwa_res - print*,'i/jafwa ',iafwa,jafwa - allocate(bitmap_afwa_global(iafwa,jafwa)) allocate(snow_dep_afwa_global(iafwa,jafwa)) @@ -625,12 +621,10 @@ subroutine readafwa use_global_afwa = .false. endif - use_nh_afwa=.false. ! use global or hemispheric files. not both. + use_nh_afwa=.false. ! Use global or hemispheric files. not both. use_sh_afwa=.false. - print*,'got here' - stop - return ! use global or hemispheric files. not both. + return ! Use global or hemispheric files. not both. else From de25b36cfb0c0fd2a9f15191ab8c1f8b6684fd64 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 5 May 2022 20:28:00 +0000 Subject: [PATCH 03/13] Add unit test. Fixes #635. --- tests/CMakeLists.txt | 1 + tests/emcsfc_snow2mdl/CMakeLists.txt | 19 ++++++++ tests/emcsfc_snow2mdl/data/config.nml | 34 +++++++++++++ tests/emcsfc_snow2mdl/ftst_program_setup.F90 | 50 ++++++++++++++++++++ 4 files changed, 104 insertions(+) create mode 100644 tests/emcsfc_snow2mdl/CMakeLists.txt create mode 100644 tests/emcsfc_snow2mdl/data/config.nml create mode 100644 tests/emcsfc_snow2mdl/ftst_program_setup.F90 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 67f3e3e0b..da5cf6e62 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -31,3 +31,4 @@ add_subdirectory(chgres_cube) add_subdirectory(fre-nctools) add_subdirectory(global_cycle) add_subdirectory(sfc_climo_gen) +add_subdirectory(emcsfc_snow2mdl) diff --git a/tests/emcsfc_snow2mdl/CMakeLists.txt b/tests/emcsfc_snow2mdl/CMakeLists.txt new file mode 100644 index 000000000..9e2929cca --- /dev/null +++ b/tests/emcsfc_snow2mdl/CMakeLists.txt @@ -0,0 +1,19 @@ +# This is the cmake build file for the tests directory of the +# UFS_UTILS project. +# +# George Gayno + +if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel)$") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8 -assume byterecl") +elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ffree-line-length-0 -fdefault-real-8") +endif() + +# Copy necessary test files from the source data directory to the +# build data directory. +execute_process( COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/data/config.nml ${CMAKE_CURRENT_BINARY_DIR}/fort.41) + +add_executable(ftst_program_setup_snow ftst_program_setup.F90) +target_link_libraries(ftst_program_setup_snow snow2mdl_lib) +add_test(NAME emcsfc_snow2mdl-ftst_program_setup COMMAND ftst_program_setup_snow) diff --git a/tests/emcsfc_snow2mdl/data/config.nml b/tests/emcsfc_snow2mdl/data/config.nml new file mode 100644 index 000000000..922fcb718 --- /dev/null +++ b/tests/emcsfc_snow2mdl/data/config.nml @@ -0,0 +1,34 @@ + &source_data + autosnow_file="autosnow.grb" + nesdis_snow_file="imssnow96.grb" + nesdis_lsmask_file="mask.grb" + afwa_snow_global_file="global_snow.grb" + afwa_snow_nh_file="NPR.SNWN.SP.S1200.MESH16" + afwa_snow_sh_file="NPR.SNWS.SP.S1200.MESH16" + afwa_lsmask_nh_file="afwa_mask.nh.bin" + afwa_lsmask_sh_file="afwa_mask.sh.bin" + / + &qc + climo_qc_file="emcsfc_snow_cover_climo.grib2" + / + &model_specs + model_lat_file="global_latitudes.t1534.3072.1536.grb" + model_lon_file="global_longitudes.t1534.3072.1536.grb" + model_lsmask_file="global_slmask.t1534.3072.1536.grb" + gfs_lpl_file="global_lonsperlat.t1534.3072.1536.txt" + / + &output_data + model_snow_file="snogrb_model" + output_grib2=.false. + / + &output_grib_time + grib_year=2012 + grib_month=10 + grib_day=29 + grib_hour=0 + / + ¶meters + lat_threshold=55.0 + min_snow_depth=0.05 + snow_cvr_threshold=50.0 + / diff --git a/tests/emcsfc_snow2mdl/ftst_program_setup.F90 b/tests/emcsfc_snow2mdl/ftst_program_setup.F90 new file mode 100644 index 000000000..f5660dc6b --- /dev/null +++ b/tests/emcsfc_snow2mdl/ftst_program_setup.F90 @@ -0,0 +1,50 @@ +program ftst_program_setup + +! Unit test for emc_snow2mdl utility, program_setup. +! +! Reads the program namelist and compares each +! variable to expected values. +! +! Author: George Gayno + + use program_setup + + implicit none + + print*, "Starting test of program_setup." + print*, "testing read_setup_namelist with file fort.41..." + + call read_config_nml + + if (trim(autosnow_file) /= "autosnow.grb") stop 2 + if (trim(nesdis_snow_file) /= "imssnow96.grb") stop 3 + if (trim(nesdis_lsmask_file) /= "mask.grb") stop 4 + if (trim(afwa_snow_global_file) /= "global_snow.grb") stop 5 + if (trim(afwa_snow_nh_file) /= "NPR.SNWN.SP.S1200.MESH16") stop 6 + if (trim(afwa_snow_sh_file) /= "NPR.SNWS.SP.S1200.MESH16") stop 7 + if (trim(afwa_lsmask_nh_file) /= "afwa_mask.nh.bin") stop 8 + if (trim(afwa_lsmask_sh_file) /= "afwa_mask.sh.bin") stop 9 + + if (trim(climo_qc_file) /= "emcsfc_snow_cover_climo.grib2") stop 10 + + if (trim(model_lat_file) /= "global_latitudes.t1534.3072.1536.grb") stop 11 + if (trim(model_lon_file) /= "global_longitudes.t1534.3072.1536.grb") stop 12 + if (trim(model_lsmask_file) /= "global_slmask.t1534.3072.1536.grb") stop 13 + if (trim(gfs_lpl_file) /= "global_lonsperlat.t1534.3072.1536.txt") stop 14 + + if (trim(model_snow_file) /= "snogrb_model") stop 15 + if (output_grib2) stop 16 + + if (grib_year /= 12) stop 17 + if (grib_month /= 10) stop 18 + if (grib_day /= 29) stop 19 + if (grib_hour /= 0) stop 20 + + if (lat_threshold /= 55.0) stop 22 + if (min_snow_depth /= 0.05) stop 23 + if (snow_cvr_threshold /= 50.0) stop 24 + + print*, "OK" + print*, "SUCCESS!" + +end program ftst_program_setup From 35e9f1f18e583e51959b8a632aced184bd925fc5 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Fri, 6 May 2022 17:50:05 +0000 Subject: [PATCH 04/13] Add LSanSuppress.supp for new unit test. Fixes #635. --- tests/emcsfc_snow2mdl/LSanSuppress.supp | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/emcsfc_snow2mdl/LSanSuppress.supp diff --git a/tests/emcsfc_snow2mdl/LSanSuppress.supp b/tests/emcsfc_snow2mdl/LSanSuppress.supp new file mode 100644 index 000000000..e69de29bb From b18c80cae58290e52c6d4a30aa7849fe48ced114 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Fri, 6 May 2022 19:26:41 +0000 Subject: [PATCH 05/13] Update LSanSuppress.supp file. Fixes #635. --- tests/emcsfc_snow2mdl/LSanSuppress.supp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/emcsfc_snow2mdl/LSanSuppress.supp b/tests/emcsfc_snow2mdl/LSanSuppress.supp index e69de29bb..a1129aaa4 100644 --- a/tests/emcsfc_snow2mdl/LSanSuppress.supp +++ b/tests/emcsfc_snow2mdl/LSanSuppress.supp @@ -0,0 +1 @@ +leak:g2 From ff520d2ac1cee5ff5df629fddc772cfdc5b832d5 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Fri, 6 May 2022 19:40:26 +0000 Subject: [PATCH 06/13] Update CMakeLists.txt file to copy LSanSuppress.supp to test directory. Fixes #635. --- tests/emcsfc_snow2mdl/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/emcsfc_snow2mdl/CMakeLists.txt b/tests/emcsfc_snow2mdl/CMakeLists.txt index 9e2929cca..ae43bf418 100644 --- a/tests/emcsfc_snow2mdl/CMakeLists.txt +++ b/tests/emcsfc_snow2mdl/CMakeLists.txt @@ -13,6 +13,8 @@ endif() # build data directory. execute_process( COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/data/config.nml ${CMAKE_CURRENT_BINARY_DIR}/fort.41) +execute_process( COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/LSanSuppress.supp ${CMAKE_CURRENT_BINARY_DIR}/LSanSuppress.supp) add_executable(ftst_program_setup_snow ftst_program_setup.F90) target_link_libraries(ftst_program_setup_snow snow2mdl_lib) From 277f48863af6da1b75d0912a9431253762a5a974 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Fri, 6 May 2022 19:58:09 +0000 Subject: [PATCH 07/13] Update prolog. Fixes #635. --- sorc/emcsfc_snow2mdl.fd/snowdat.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/emcsfc_snow2mdl.fd/snowdat.F90 b/sorc/emcsfc_snow2mdl.fd/snowdat.F90 index 1ef788f18..7ffe71944 100755 --- a/sorc/emcsfc_snow2mdl.fd/snowdat.F90 +++ b/sorc/emcsfc_snow2mdl.fd/snowdat.F90 @@ -509,7 +509,7 @@ end subroutine readnesdis !> Read snow depth data and masks. !! -!! @note Read nh and sh afwa snow depth data and +!! @note Read afwa snow depth data and !! land sea mask. !! !! program history log: From 0774b7f1631e521b4e29dffa311bb51239dd98d5 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 18 May 2022 19:03:38 +0000 Subject: [PATCH 08/13] Add regression test for new global afwa snow data. Update Hera driver script. Fixes #635 --- reg_tests/snow2mdl/driver.hera.sh | 46 +++++++++------ reg_tests/snow2mdl/snow2mdl.global.sh | 57 +++++++++++++++++++ .../snow2mdl/{snow2mdl.sh => snow2mdl.ops.sh} | 15 ++--- ush/emcsfc_snow.sh | 4 +- 4 files changed, 95 insertions(+), 27 deletions(-) create mode 100755 reg_tests/snow2mdl/snow2mdl.global.sh rename reg_tests/snow2mdl/{snow2mdl.sh => snow2mdl.ops.sh} (71%) diff --git a/reg_tests/snow2mdl/driver.hera.sh b/reg_tests/snow2mdl/driver.hera.sh index 11e854954..bc80942d8 100755 --- a/reg_tests/snow2mdl/driver.hera.sh +++ b/reg_tests/snow2mdl/driver.hera.sh @@ -2,12 +2,12 @@ #----------------------------------------------------------------------------- # -# Run snow2mdl consistency test on Hera. +# Run snow2mdl consistency tests on Hera. # -# Set $DATA to your working directory. Set the project code (SBATCH -A) +# Set $DATA_ROOT to your working directory. Set the project code (SBATCH -A) # and queue (SBATCH -q) as appropriate. # -# Invoke the script as follows: sbatch $script +# Invoke the script from the command line as follows: ./$script # # Log output is placed in consistency.log. A summary is # placed in summary.log @@ -18,15 +18,6 @@ # #----------------------------------------------------------------------------- -#SBATCH -J snow -#SBATCH -A fv3-cpu -#SBATCH --open-mode=truncate -#SBATCH -o consistency.log -#SBATCH -e consistency.log -#SBATCH --ntasks=1 -#SBATCH -q debug -#SBATCH -t 00:03:00 - set -x compiler=${compiler:-"intel"} @@ -36,8 +27,13 @@ module use ../../modulefiles module load build.$target.$compiler module list -export DATA="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}" -export DATA="${DATA}/reg-tests/snow2mdl" +DATA_ROOT="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}" +DATA_ROOT="${DATA_ROOT}/reg-tests/snow2mdl" + +rm -fr $DATA_ROOT + +PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}" +QUEUE="${QUEUE:-batch}" #----------------------------------------------------------------------------- # Should not have to change anything below. @@ -50,13 +46,29 @@ if [ "$UPDATE_BASELINE" = "TRUE" ]; then source ../get_hash.sh fi -rm -fr $DATA - export HOMEreg=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/snow2mdl export HOMEgfs=$PWD/../.. export WGRIB=/scratch2/NCEPDEV/nwprod/NCEPLIBS/utils/grib_util.v1.1.1/exec/wgrib export WGRIB2=/scratch2/NCEPDEV/nwprod/NCEPLIBS/utils/grib_util.v1.1.1/exec/wgrib2 -./snow2mdl.sh +# The first test mimics GFS OPS. + +export DATA="${DATA_ROOT}/test.ops" +TEST1=$(sbatch --parsable -J snow.ops -A ${PROJECT_CODE} -o consistency.log -e consistency.log \ + --ntasks=1 -q ${QUEUE} -t 00:03:00 ./snow2mdl.ops.sh) + +# The second test is for the new AFWA global GRIB2 data. + +export DATA="${DATA_ROOT}/test.global" +TEST2=$(sbatch --parsable -J snow.global -A ${PROJECT_CODE} -o consistency.log -e consistency.log \ + --ntasks=1 -q ${QUEUE} -t 00:03:00 -d afterok:$TEST1 ./snow2mdl.global.sh) + +# Create summary file. + +sbatch --nodes=1 -t 0:01:00 -A ${PROJECT_CODE} -J snow_summary -o consistency.log -e consistency.log \ + --open-mode=append -q ${QUEUE} -d afterok:$TEST2 << EOF +#!/bin/bash +grep -a '<<<' consistency.log > summary.log +EOF exit 0 diff --git a/reg_tests/snow2mdl/snow2mdl.global.sh b/reg_tests/snow2mdl/snow2mdl.global.sh new file mode 100755 index 000000000..77b9b934d --- /dev/null +++ b/reg_tests/snow2mdl/snow2mdl.global.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +#-------------------------------------------------------------------------- +# Create a snow file from afwa global data. This script is run from +# its machine-specific driver. +#-------------------------------------------------------------------------- + +set -x + +export IMS_FILE=$HOMEreg/input_data/imssnow96.afwa.global.grb +export AFWA_NH_FILE="" +export AFWA_SH_FILE="" +export AFWA_GLOBAL_FILE="$HOMEreg/input_data/afwa.global.grb2" + +export MODEL_LATITUDE_FILE=$HOMEgfs/fix/fix_am/global_latitudes.t1534.3072.1536.grb +export MODEL_LONGITUDE_FILE=$HOMEgfs/fix/fix_am/global_longitudes.t1534.3072.1536.grb +export MODEL_SLMASK_FILE=$HOMEgfs/fix/fix_am/global_slmask.t1534.3072.1536.grb +export GFS_LONSPERLAT_FILE=$HOMEgfs/fix/fix_am/global_lonsperlat.t1534.3072.1536.txt + +export OMP_NUM_THREADS=1 +export OUTPUT_GRIB2=.false. + +${HOMEgfs}/ush/emcsfc_snow.sh + +iret=$? +if [ $iret -ne 0 ]; then + set +x + echo "<<< SNOW2MDL GLOBAL TEST FAILED. <<<" + exit $iret +fi + +test_failed=0 + +cmp ${DATA}/snogrb_model $HOMEreg/baseline_data/t1534.global/snogrb_model +iret=$? +if [ $iret -ne 0 ]; then + test_failed=1 +fi + +set +x +if [ $test_failed -ne 0 ]; then + echo + echo "*********************************" + echo "<<< SNOW2MDL GLOBAL TEST FAILED. >>>" + echo "*********************************" + if [ "$UPDATE_BASELINE" = "TRUE" ]; then + cd $DATA + $HOMEgfs/reg_tests/update_baseline.sh $HOMEreg "t1534.global" $commit_num + fi +else + echo + echo "*********************************" + echo "<<< SNOW2MDL GLOBAL TEST PASSED. >>>" + echo "*********************************" +fi + +exit diff --git a/reg_tests/snow2mdl/snow2mdl.sh b/reg_tests/snow2mdl/snow2mdl.ops.sh similarity index 71% rename from reg_tests/snow2mdl/snow2mdl.sh rename to reg_tests/snow2mdl/snow2mdl.ops.sh index 4743ad42b..700839a3b 100755 --- a/reg_tests/snow2mdl/snow2mdl.sh +++ b/reg_tests/snow2mdl/snow2mdl.ops.sh @@ -1,7 +1,7 @@ #!/bin/bash #-------------------------------------------------------------------------- -# Create a T1534 snow file. This script is run from its machine-specific +# Mimic GFS OPS. This script is run from its machine-specific # driver. #-------------------------------------------------------------------------- @@ -24,14 +24,13 @@ ${HOMEgfs}/ush/emcsfc_snow.sh iret=$? if [ $iret -ne 0 ]; then set +x - echo "<<< SNOW2MDL TEST FAILED. <<<" - echo "<<< SNOW2MDL TEST FAILED. <<<" > ./summary.log + echo "<<< SNOW2MDL OPS TEST FAILED. <<<" exit $iret fi test_failed=0 -cmp ${DATA}/snogrb_model $HOMEreg/baseline_data/t1534/snogrb_model +cmp ${DATA}/snogrb_model $HOMEreg/baseline_data/t1534.ops/snogrb_model iret=$? if [ $iret -ne 0 ]; then test_failed=1 @@ -41,19 +40,17 @@ set +x if [ $test_failed -ne 0 ]; then echo echo "*********************************" - echo "<<< SNOW2MDL TEST FAILED. >>>" + echo "<<< SNOW2MDL OPS TEST FAILED. >>>" echo "*********************************" - echo "<<< SNOW2MDL TEST FAILED. >>>" > ./summary.log if [ "$UPDATE_BASELINE" = "TRUE" ]; then cd $DATA - $HOMEgfs/reg_tests/update_baseline.sh $HOMEreg "t1534" $commit_num + $HOMEgfs/reg_tests/update_baseline.sh $HOMEreg "t1534.ops" $commit_num fi else echo echo "*********************************" - echo "<<< SNOW2MDL TEST PASSED. >>>" + echo "<<< SNOW2MDL OPS TEST PASSED. >>>" echo "*********************************" - echo "<<< SNOW2MDL TEST PASSED. >>>" > ./summary.log fi exit diff --git a/ush/emcsfc_snow.sh b/ush/emcsfc_snow.sh index 7c4c6c429..a844cc317 100755 --- a/ush/emcsfc_snow.sh +++ b/ush/emcsfc_snow.sh @@ -24,6 +24,7 @@ # $MODEL_LONGITUDE_FILE - model longitude (grib 1 or 2) # $AFWA_NH_FILE - nh afwa snow data (grib 1) # $AFWA_SH_FILE - sh afwa snow data (grib 1) +# $AFWA_GLOBAL_FILE - global afwa snow data (grib 2) # $IMS_FILE - nh ims snow cover data (grib 2) # $CLIMO_QC - nh climatological snow cover (grib 2) # fort.41 - program configuration namelist @@ -108,6 +109,7 @@ GFS_LONSPERLAT_FILE=${GFS_LONSPERLAT_FILE:-global_lonsperlat.t1534.3072.1536.txt AFWA_NH_FILE=${AFWA_NH_FILE:-"NPR.SNWN.SP.S1200.MESH16"} AFWA_SH_FILE=${AFWA_SH_FILE:-"NPR.SNWS.SP.S1200.MESH16"} +AFWA_GLOBAL_FILE=${AFWA_GLOBAL_FILE:-""} IMS_FILE=${IMS_FILE:-"imssnow96.grb.grib2"} #------------------------------------------------------------------------ @@ -185,7 +187,7 @@ cat > ./fort.41 << ! autosnow_file="" nesdis_snow_file="${IMS_FILE}" nesdis_lsmask_file="" - afwa_snow_global_file="" + afwa_snow_global_file="${AFWA_GLOBAL_FILE}" afwa_snow_nh_file="${AFWA_NH_FILE}" afwa_snow_sh_file="${AFWA_SH_FILE}" afwa_lsmask_nh_file="" From 1a7cf7c85d51b9b998151f59788f82c8fffc1a8c Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 18 May 2022 20:48:23 +0000 Subject: [PATCH 09/13] Update Jet script. Fixes #635. --- reg_tests/snow2mdl/driver.jet.sh | 43 +++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/reg_tests/snow2mdl/driver.jet.sh b/reg_tests/snow2mdl/driver.jet.sh index cd06cf513..87ab421e5 100755 --- a/reg_tests/snow2mdl/driver.jet.sh +++ b/reg_tests/snow2mdl/driver.jet.sh @@ -2,12 +2,12 @@ #----------------------------------------------------------------------------- # -# Run snow2mdl consistency test on Jet. +# Run snow2mdl consistency tests on Jet. # -# Set $DATA to your working directory. Set the project code (SBATCH -A) -# and queue (SBATCH -q) as appropriate. +# Set $DATA_ROOT to your working directory. Set the project code and +# and queue as appropriate. # -# Invoke the script as follows: sbatch $script +# Invoke the script as follows: ./$script # # Log output is placed in consistency.log. A summary is # placed in summary.log @@ -18,13 +18,6 @@ # #----------------------------------------------------------------------------- -#SBATCH --nodes=1 -#SBATCH --partition=sjet -#SBATCH --time 0:01 -#SBATCH --account=emcda -#SBATCH --job-name=snow2mdl -#SBATCH -o consistency.log -#SBATCH -e consistency.log set -x @@ -33,8 +26,11 @@ module use ../../modulefiles module load build.$target.intel module list -export DATA="${WORK_DIR:-/lfs4/HFIP/emcda/$LOGNAME/stmp}" -export DATA="${DATA}/reg-tests/snow2mdl" +DATA_ROOT="${WORK_DIR:-/lfs4/HFIP/emcda/$LOGNAME/stmp}" +DATA_ROOT="${DATA_ROOT}/reg-tests/snow2mdl" + +PROJECT_CODE="${PROJECT_CODE:-hfv3gfs}" +QUEUE="${QUEUE:-batch}" #----------------------------------------------------------------------------- # Should not have to change anything below. @@ -52,8 +48,25 @@ export HOMEgfs=$PWD/../.. export WGRIB=/apps/wgrib/1.8.1.0b/bin/wgrib export WGRIB2=/apps/wgrib2/0.1.9.6a/bin/wgrib2 -rm -fr $DATA +rm -fr $DATA_ROOT + +# This tests the OPS GFS snow processing. + +export DATA="${DATA_ROOT}/test.ops" +TEST1=$(sbatch --parsable --nodes=1 --partition=xjet --time 0:02 -J snow.ops -o consistency.log \ + -e consistency.log -A $PROJECT_CODE -q $QUEUE ./snow2mdl.ops.sh) -./snow2mdl.sh +# Test the new global afwa grib2 data. +export DATA="${DATA_ROOT}/test.global" +TEST2=$(sbatch --parsable --nodes=1 --partition=xjet --time 0:02 -J snow.global -o consistency.log \ + -e consistency.log -A $PROJECT_CODE -q $QUEUE -d afterok:$TEST1 ./snow2mdl.global.sh) + +# Create summary file. + +sbatch --nodes=1 --partition=xjet -t 0:01:00 -A $PROJECT_CODE -J snow.summary -o consistency.log \ + -e consistency.log --open-mode=append -q $QUEUE -d afterok:$TEST2 << EOF +#!/bin/bash +grep -a '<<<' consistency.log > summary.log +EOF exit 0 From bf2ea4d890d13e0b9269792f87043feec26fc9b7 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 19 May 2022 09:48:27 -0500 Subject: [PATCH 10/13] Update Orion script. Fixes #635 --- reg_tests/snow2mdl/driver.orion.sh | 49 +++++++++++++++++++----------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/reg_tests/snow2mdl/driver.orion.sh b/reg_tests/snow2mdl/driver.orion.sh index 99ad0c14f..4715557cb 100755 --- a/reg_tests/snow2mdl/driver.orion.sh +++ b/reg_tests/snow2mdl/driver.orion.sh @@ -2,31 +2,22 @@ #----------------------------------------------------------------------------- # -# Run snow2mdl consistency test on Orion. +# Run snow2mdl consistency tests on Orion. # -# Set $DATA to your working directory. Set the project code (SBATCH -A) -# and queue (SBATCH -q) as appropriate. +# Set $DATA_ROOT to your working directory. Set the project code +# and queue as appropriate. # -# Invoke the script as follows: sbatch $script +# Invoke the script as follows: ./$script # # Log output is placed in consistency.log. A summary is # placed in summary.log # # The test fails when its output does not match the baseline file -# as determined by the 'cmp' command. The baseline file is +# as determined by the 'cmp' command. The baseline files are # stored in HOMEreg. # #----------------------------------------------------------------------------- -#SBATCH -J snow -#SBATCH -A fv3-cpu -#SBATCH --open-mode=truncate -#SBATCH -o consistency.log -#SBATCH -e consistency.log -#SBATCH --ntasks=1 -#SBATCH -q debug -#SBATCH -t 00:03:00 - set -x source ../../sorc/machine-setup.sh > /dev/null 2>&1 @@ -36,8 +27,11 @@ module list ulimit -s unlimited -export DATA="${WORK_DIR:-/work/noaa/stmp/$LOGNAME}" -export DATA="${DATA}/reg-tests/snow2mdl" +export DATA_ROOT="${WORK_DIR:-/work/noaa/stmp/$LOGNAME}" +export DATA_ROOT="${DATA_ROOT}/reg-tests/snow2mdl" + +PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}" +QUEUE="${QUEUE:-batch}" #----------------------------------------------------------------------------- # Should not have to change anything below. @@ -50,13 +44,32 @@ if [ "$UPDATE_BASELINE" = "TRUE" ]; then source ../get_hash.sh fi -rm -fr $DATA +rm -fr $DATA_ROOT export HOMEreg=/work/noaa/nems/role-nems/ufs_utils/reg_tests/snow2mdl export HOMEgfs=$PWD/../.. export WGRIB=/apps/contrib/NCEPLIBS/orion/utils/grib_util.v1.2.0/exec/wgrib export WGRIB2=/apps/contrib/NCEPLIBS/orion/utils/grib_util.v1.2.0/exec/wgrib2 -./snow2mdl.sh +# The first test mimics GFS OPS. + +export DATA="${DATA_ROOT}/test.ops" +TEST1=$(sbatch --parsable -J snow.ops -A $PROJECT_CODE -o consistency.log \ + -e consistency.log --ntasks=1 -q $QUEUE -t 00:03:00 ./snow2mdl.ops.sh) + +# This tests the afwa global grib2 data. + +export DATA="${DATA_ROOT}/test.global" +TEST2=$(sbatch --parsable -J snow.global -A $PROJECT_CODE -o consistency.log \ + -e consistency.log --ntasks=1 -q $QUEUE -t 00:03:00 --open-mode=append \ + -d afterok:$TEST1 ./snow2mdl.global.sh) + +# Create the summary file. + +sbatch --ntasks=1 -t 0:01:00 -A $PROJECT_CODE -J snow_summary -o consistency.log -e consistency.log \ + --open-mode=append -q $QUEUE -d afterok:$TEST2 << EOF +#!/bin/bash +grep -a '<<<' consistency.log > summary.log +EOF exit 0 From b40895b763bd2f4956168e0d5c372230edc450a3 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 19 May 2022 19:17:48 +0000 Subject: [PATCH 11/13] Update wcoss-dell script. Fixes #635. --- reg_tests/snow2mdl/driver.wcoss_dell_p3.sh | 41 +++++++++++++--------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/reg_tests/snow2mdl/driver.wcoss_dell_p3.sh b/reg_tests/snow2mdl/driver.wcoss_dell_p3.sh index f1e417587..89fd5b4ec 100755 --- a/reg_tests/snow2mdl/driver.wcoss_dell_p3.sh +++ b/reg_tests/snow2mdl/driver.wcoss_dell_p3.sh @@ -2,30 +2,22 @@ #----------------------------------------------------------------------------- # -# Run snow2mdl consistency test on WCOSS-Dell. +# Run snow2mdl consistency tests on WCOSS-Dell. # -# Set $DATA to your working directory. Set the project code (BSUB -P) -# and queue (BSUB -q) as appropriate. +# Set $DATA_ROOT to your working directory. Set the project code +# and queue as appropriate. # -# Invoke the script as follows: cat $script | bsub +# Invoke the script as follows: ./$script # # Log output is placed in consistency.log. A summary is # placed in summary.log # # The test fails when its output does not match the baseline file -# as determined by the 'cmp' command. The baseline file is +# as determined by the 'cmp' command. The baseline files are # stored in HOMEreg. # #----------------------------------------------------------------------------- -#BSUB -W 0:02 -#BSUB -o consistency.log -#BSUB -e consistency.log -#BSUB -J s2m_regt -#BSUB -q debug -#BSUB -R "affinity[core(1)]" -#BSUB -P GFS-DEV - source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.intel @@ -34,8 +26,11 @@ module list set -x -export DATA="${WORK_DIR:-/gpfs/dell1/stmp/$LOGNAME}" -export DATA="${DATA}/reg-tests/snow2mdl" +export DATA_ROOT="${WORK_DIR:-/gpfs/dell1/stmp/$LOGNAME}" +export DATA_ROOT="${DATA_ROOT}/reg-tests/snow2mdl" + +PROJECT_CODE=${PROJECT_CODE:-"GFS-DEV"} +QUEUE=${QUEUE:-"debug"} #----------------------------------------------------------------------------- # Should not have to change anything below. @@ -53,8 +48,20 @@ export HOMEgfs=$PWD/../.. export WGRIB=/gpfs/dell1/nco/ops/nwprod/grib_util.v1.0.6/exec/wgrib export WGRIB2=/gpfs/dell1/nco/ops/nwprod/grib_util.v1.0.6/exec/wgrib2 -rm -fr $DATA +LOG_FILE=consistency.log +SUM_FILE=summary.log + +rm -fr $DATA_ROOT + +export DATA=$DATA_ROOT/test.ops +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J snow.ops -W 0:02 \ + -R "affinity[core(1)]" "$PWD/snow2mdl.ops.sh" + +export DATA=$DATA_ROOT/test.global +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J snow.global -W 0:02 \ + -R "affinity[core(1)]" -w 'ended(snow.ops)' "$PWD/snow2mdl.global.sh" -./snow2mdl.sh +bsub -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J summary -R "affinity[core(1)]" -R "rusage[mem=100]" -W 0:01 \ + -w 'ended(snow.global)' "grep -a '<<<' $LOG_FILE >> $SUM_FILE" exit 0 From 5a04ed89adb3165bb47fc8d0fd030d74ce775018 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 19 May 2022 20:03:44 +0000 Subject: [PATCH 12/13] Update wcoss-cray script. Fixes #635. --- reg_tests/snow2mdl/driver.wcoss_cray.sh | 47 ++++++++++++++++--------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/reg_tests/snow2mdl/driver.wcoss_cray.sh b/reg_tests/snow2mdl/driver.wcoss_cray.sh index c0dc69509..2b92d4fa5 100755 --- a/reg_tests/snow2mdl/driver.wcoss_cray.sh +++ b/reg_tests/snow2mdl/driver.wcoss_cray.sh @@ -2,30 +2,22 @@ #----------------------------------------------------------------------------- # -# Run snow2mdl consistency test on WCOSS-Cray. +# Run snow2mdl consistency tests on WCOSS-Cray. # -# Set $DATA to your working directory. Set the project code (BSUB -P) -# and queue (BSUB -q) as appropriate. +# Set $DATA_ROOT to your working directory. Set the project code +# and queue as appropriate. # -# Invoke the script as follows: cat $script | bsub +# Invoke the script as follows: ./$script # # Log output is placed in consistency.log. A summary is # placed in summary.log # # The test fails when its output does not match the baseline file -# as determined by the 'cmp' command. The baseline file is +# as determined by the 'cmp' command. The baseline files are # stored in HOMEreg. # #----------------------------------------------------------------------------- -#BSUB -W 0:02 -#BSUB -o consistency.log -#BSUB -e consistency.log -#BSUB -J s2m_regt -#BSUB -q debug -#BSUB -R "rusage[mem=2000]" -#BSUB -P GFS-DEV - set -x source ../../sorc/machine-setup.sh > /dev/null 2>&1 @@ -33,8 +25,8 @@ module use ../../modulefiles module load build.$target.intel module list -export DATA="${WORK_DIR:-/gpfs/hps3/stmp/$LOGNAME}" -export DATA="${DATA}/reg-tests/snow2mdl" +DATA_ROOT="${WORK_DIR:-/gpfs/hps3/stmp/$LOGNAME}" +DATA_ROOT="${DATA_ROOT}/reg-tests/snow2mdl" #----------------------------------------------------------------------------- # Should not have to change anything below. @@ -47,13 +39,34 @@ if [ "$UPDATE_BASELINE" = "TRUE" ]; then source ../get_hash.sh fi +PROJECT_CODE=${PROJECT_CODE:-GFS-DEV} +QUEUE=${QUEUE:-dev} + export HOMEreg=/gpfs/hps3/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/snow2mdl export HOMEgfs=$PWD/../.. export WGRIB=/gpfs/hps/nco/ops/nwprod/grib_util.v1.0.2/exec/wgrib export WGRIB2=/gpfs/hps/nco/ops/nwprod/grib_util.v1.0.2/exec/wgrib2 -rm -fr $DATA +rm -fr $DATA_ROOT + +LOG_FILE="consistency.log" +SUM_FILE="summary.log" + +# Test the ops function of snow2mdl. + +export DATA=$DATA_ROOT/test.ops +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J snow.ops -W 0:02 \ + -R "rusage[mem=2000]" "$PWD/snow2mdl.ops.sh" + +# Test the afwa global snow data. + +export DATA=$DATA_ROOT/test.global +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J snow.global -W 0:02 \ + -R "rusage[mem=2000]" -w 'ended(snow.ops)' "$PWD/snow2mdl.global.sh" + +# Create a summary file. -./snow2mdl.sh +bsub -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J summary -R "rusage[mem=100]" -W 0:01 \ + -w 'ended(snow.global)' "grep -a '<<<' $LOG_FILE >> $SUM_FILE" exit 0 From 224bf66dbf590e24ae174b585e80472256a50105 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 19 May 2022 20:31:53 +0000 Subject: [PATCH 13/13] Update rt.sh for changes to snow test. Fixes #635. --- reg_tests/rt.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/reg_tests/rt.sh b/reg_tests/rt.sh index dd25f276a..bf196f576 100755 --- a/reg_tests/rt.sh +++ b/reg_tests/rt.sh @@ -68,7 +68,7 @@ cd fix cd ../reg_tests sleep_time=0 -for dir in global_cycle chgres_cube grid_gen; do +for dir in snow2mdl global_cycle chgres_cube grid_gen; do cd $dir ./driver.$target.sh # Wait for job to complete @@ -89,8 +89,7 @@ elif [[ $target == "wcoss_cray" ]]; then module load xt-lsfhpc/9.1.3 fi - -for dir in snow2mdl ice_blend; do +for dir in ice_blend; do cd $dir if [[ $target == "hera" ]] || [[ $target == "jet" ]] || [[ $target == "orion" ]]; then sbatch -A ${PROJECT_CODE} ./driver.$target.sh