Move ESMF-based surface regridding code from GDASApp to UFS_UTILS

CMakeLists.txt

@@ -20,6 +20,7 @@ option(ENABLE_DOCS "Enable generation of doxygen-based documentation." OFF)
 option(ICEBLEND "Enable building emcsfc_ice_blend.fd" ON)
 option(SNOW2MDL "Enable building emcsfc_snow2mdl.fd" ON)
 option(GCYCLE "Enable building global_cycle.fd" ON)
+option(REGRID "Enable building regrid_sfc.fd" ON)
 option(FRENCTOOLS "Enable building fre-nctools.fd" ON)
 option(GRIDTOOLS "Enable building grid_tools.fd" ON)
 option(CHGRES "Enable building chgres_cube.fd" ON)

README.md

@@ -25,6 +25,7 @@ gdas_init | George Gayno
 global_cycle | George Gayno, Shrinivas Moorthi, Xu Li
 grid_tools | R. J. Purser (regional_esg_grid), Ben Blake (shave.fd), Gerard Ketefian (global_equiv_resol), Tsukasa Fujita, JMA (pmat2), GFDL programmer (topo filtering code).
 orog_mask_tools | Ning Wang, Jordan Alpert, Shan Sun and Ning Wang
+regrid_sfc | Clara Draper
 sfc_climo_gen | George Gayno
 vcoord_gen | Fanglin Yang
 weight_gen | George Gayno

docs/main.md

@@ -83,6 +83,10 @@ https://github.com/ufs-community/UFS_UTILS.
   surface climatological fields, such as vegetation type and albedo,
   for an FV3 grid.
 
+- <a href="regrid_sfc/index.html">regrid_sfc</a> - Masked
+   bilinear interpolation of increments and states between fv3 and 
+   Gaussian grids.
+
 - <a href="vcoord_gen/index.html">vcoord_gen</a> - Generates hybrid
   coordinate parameters from fields such as surface pressure, model
   top and the number of vertical levels. Outputs the 'ak' and 'bk'

reg_tests/regrid_sfc/driver.sh

@@ -0,0 +1,123 @@
+#!/bin/bash
+
+#-----------------------------------------------------------------------------
+#
+# Run regrid_sfc consistency tests.
+#
+# Set $WORK_DIR to your working directory. 
+# Set the $PROJECT_CODE and $QUEUE as appropriate.
+#
+# Invoke the script from command line as follows:  ./$script
+#
+# Log output is placed in consistency.log??.  A summary is
+# placed in summary.log
+#
+# A test fails when its output does not match the baseline files
+# as determined by the 'nccmp' utility. The baseline files are
+# stored in HOMEreg.
+#
+#-----------------------------------------------------------------------------
+
+set -x
+
+source ../../sorc/machine-setup.sh > /dev/null 2>&1
+module use ../../modulefiles
+compiler=${compiler:-intelllvm}
+if [[ "$compiler" == "intelllvm" ]]; then
+  if [[ ! -f ../../modulefiles/build.$target.$compiler.lua ]];then
+     set +x
+     echo "IntelLLVM not available. Will use Intel Classic."
+     set -x
+    compiler=intel
+  fi
+fi
+module load build.$target.$compiler
+if [[ "$target" == "wcoss2" ]];then
+  module load netcdf
+  module load nccmp
+fi
+set +x
+module list
+set -x
+
+export UPDATE_BASELINE="FALSE"
+#export UPDATE_BASELINE="TRUE"
+
+if [[ "$UPDATE_BASELINE" == "TRUE" ]]; then
+  source ../get_hash.sh
+fi
+
+if [[ "$target" == "jet" ]];then
+  export WORK_DIR="${WORK_DIR:-/lfs5/HFIP/emcda/$LOGNAME/stmp}"
+  PROJECT_CODE="${PROJECT_CODE:-hfv3gfs}"
+  QUEUE="${QUEUE:-batch}"
+  export HOMEreg=/lfs5/HFIP/hfv3gfs/emc.nemspara/role.ufsutils/ufs_utils/reg_tests/regrid_sfc
+  export APRUN_REGRID=srun
+  PARTITION=xjet
+elif [[ "$target" == "hera" ]];then
+  WORK_DIR="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}"
+  PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}"
+  QUEUE="${QUEUE:-batch}"
+  export HOMEreg=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/regrid_sfc/
+  export APRUN_REGRID=srun
+  PARTITION=hera
+elif [[ "$target" == "orion" ]];then
+  WORK_DIR="${WORK_DIR:-/work/noaa/stmp/$LOGNAME}"
+  PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}"
+  QUEUE="${QUEUE:-batch}"
+  export HOMEreg=/work/noaa/nems/role-nems/ufs_utils/reg_tests/regrid_sfc
+  export APRUN_REGRID=srun
+  PARTITION=orion
+elif [[ "$target" == "hercules" ]];then
+  WORK_DIR="${WORK_DIR:-/work2/noaa/stmp/$LOGNAME}"
+  PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}"
+  QUEUE="${QUEUE:-batch}"
+  export HOMEreg=/work/noaa/nems/role-nems/ufs_utils.hercules/reg_tests/regrid_sfc
+  export APRUN_REGRID=srun
+  PARTITION=hercules
+elif [[ "$target" == "wcoss2" ]];then
+  WORK_DIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}"
+  PROJECT_CODE="${PROJECT_CODE:-GFS-DEV}"
+  QUEUE="${QUEUE:-dev}"
+  export HOMEreg=/lfs/h2/emc/nems/noscrub/emc.nems/UFS_UTILS/reg_tests/regrid_sfc
+  export APRUN_REGRID="mpiexec -n 6 -ppn 6 --cpu-bind core"
+fi
+
+DATA_DIR="${WORK_DIR}/reg-tests/regrid_sfc"
+export NWPROD=$PWD/../..
+
+LOG_FILE=consistency.log01
+rm -f $LOG_FILE
+export DATA="${DATA_DIR}/test1"
+if [[ "$target" == "wcoss2" ]];then
+  TEST1=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \
+        -N gauss2fv3incr -l select=1:ncpus=6:ompthreads=1:mem=10GB ./gauss2fv3incr.sh)
+else
+  TEST1=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J gauss2fv3incr \
+      --partition=$PARTITION -o $LOG_FILE -e $LOG_FILE ./gauss2fv3incr.sh)
+fi
+
+LOG_FILE=consistency.log
+rm -f $LOG_FILE summary.log
+
+if [[ "$target" == "wcoss2" ]];then
+
+this_dir=$PWD
+qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:01:00 \
+        -N summary -l select=1:ncpus=1:mem=100MB -W depend=afterok:$TEST1 << EOF
+#!/bin/bash
+cd $this_dir
+grep -a '<<<' ${LOG_FILE}?? | grep -v echo > ./summary.log
+EOF
+
+else
+
+sbatch --partition=$PARTITION --nodes=1  -t 0:01:00 -A $PROJECT_CODE -J summary -o $LOG_FILE -e $LOG_FILE \
+       --open-mode=append -q $QUEUE -d afterok:$TEST1 << EOF
+#!/bin/bash
+grep -a '<<<' ${LOG_FILE}* > ./summary.log
+EOF
+
+fi
+
+exit

reg_tests/regrid_sfc/gauss2fv3incr.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+
+#------------------------------------------------------------------
+# Run regrid_sfc utility to regrid a gsi (Gaussian) increment 
+# to the cube sphere grid.
+# Compare output to a baseline set of files using the 'nccmp' 
+# utility.
+#------------------------------------------------------------------
+
+set -x
+
+NCCMP=${NCCMP:-$(which nccmp)}
+
+REGRID_EXEC=${NWPROD}/exec/regridStates.x
+FIXorog=$HOMEreg/fix
+
+COMIN_REGTEST=$HOMEreg/input_data_noahmp
+
+# resolutions
+CASE_IN=C192
+CASE_OUT=C192
+OCNRES_OUT=050
+
+LONB_CASE_IN=$((4*${CASE_IN:1}))
+LATB_CASE_IN=$((2*${CASE_IN:1}))
+
+# create clean data dir 
+if [[ -d $DATA ]]; then
+   rm -rf ${DATA}
+fi
+
+mkdir -p $DATA
+
+cd ${DATA}
+
+# input, increment
+/bin/cp -p ${COMIN_REGTEST}/sfcincr_gsi ${DATA}/sfcincr_gsi
+
+# input, fixed files
+ln -sf "${FIXorog}/gaussian.${LONB_CASE_IN}.${LATB_CASE_IN}.nc" \
+        "${DATA}/gaussian_scrip.nc"
+
+# output, fixed files
+ln -sf "${FIXorog}/${CASE_OUT}_mosaic.nc" \
+        "${DATA}/${CASE_OUT}_mosaic.nc"
+
+ntiles=6
+for n in $(seq 1 $ntiles); do
+    ln -sf ${FIXorog}/${CASE_OUT}.mx${OCNRES_OUT}.vegetation_type.tile${n}.nc  ${DATA}/vegetation_type.tile${n}.nc
+    ln -sf ${FIXorog}/${CASE_OUT}_grid.tile${n}.nc ${DATA}/${CASE_OUT}_grid.tile${n}.nc
+done
+
+# namelist
+cat << EOF > regrid.nml
+ &config
+  n_vars=4,
+  variable_list="soilt1_inc", "soilt2_inc", "slc1_inc", "slc2_inc",
+  missing_value=0.,
+  extrap_levs=2,
+ /
+ &input
+  gridtype="gau_inc",
+  ires=${LONB_CASE_IN},
+  jres=${LATB_CASE_IN},
+  fname="sfcincr_gsi",
+  dir="./",
+  fname_coord="gaussian_scrip.nc",
+  dir_coord="./"
+ /
+
+ &output
+  gridtype="fv3_rst",
+  ires=${CASE_OUT:1},
+  jres=${CASE_OUT:1},
+  fname="sfci",
+  dir="./",
+  fname_mask="vegetation_type"
+  dir_mask="./"
+  dir_coord="./",
+ /
+EOF
+
+# run the executable
+$APRUN_REGRID $REGRID_EXEC
+
+iret=$?
+if [ $iret -ne 0 ]; then
+  set +x
+  echo "<<< REGRID SFC TEST FAILED. >>>"
+  exit $iret
+fi
+
+test_failed=0
+
+# check the ouput
+for files in sfci.tile*
+do
+  if [ -f $files ]; then
+    echo CHECK $files
+    $NCCMP -dmfqS $files $HOMEreg/baseline_data/gauss2fv3incr/$files
+    iret=$?
+    if [ $iret -ne 0 ]; then
+      test_failed=1
+    fi
+  fi
+done
+
+set +x
+if [ $test_failed -ne 0 ]; then
+  echo
+  echo "**********************************************"
+  echo "<<< REGRID SFC TEST FAILED. >>>"
+  echo "**********************************************"
+  if [ "$UPDATE_BASELINE" = "TRUE" ]; then
+# Remove files from work directory that we don't want copied 
+# to the baseline directory.
+    for files in *
+    do
+      if [[ -f $files ]];then
+        file_check=$(echo $files | cut -c 1-4)
+        if [[ "$file_check" != "sfci" ]];then
+          rm -f $files
+        fi
+      fi
+    done
+    ${NWPROD}/reg_tests/update_baseline.sh $HOMEreg "gauss2fv3incr" $commit_num
+  fi
+else
+  echo
+  echo "*****************************************"
+  echo "<<< REGRID SFC TEST PASSED >>>"
+  echo "*****************************************"
+fi
+
+exit

reg_tests/rt.sh

@@ -76,10 +76,28 @@ cd ../reg_tests
 #if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hera" ]] || [[ $target == "hercules" ]] || [[ $target == "wcoss2" ]] ; then
 if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hera" ]] || [[ $target == "hercules" ]] ; then
 
-  cd ocnice_prep
+  cd regrid_sfc
   export ACCOUNT=$PROJECT_CODE
   export STMP=$WORK_DIR/reg-tests
 
+  ./driver.sh
+
+  set -x
+
+  sleep_time=0
+  while [ ! -f "summary.log" ]; do
+    sleep 10
+    sleep_time=$((sleep_time+10))
+    if (( sleep_time > TIMEOUT_LIMIT )); then
+       kill -9 %1
+       mail -s "UFS_UTILS Consistency Tests timed out on ${target}" ${MAILTO} < ${WORK_DIR}/reg_test_results.txt
+       exit 1
+    fi
+  done
+  cd ..
+
+  cd ocnice_prep
+
   ./rt.sh 2>/dev/null &
 
   set -x
@@ -97,8 +115,6 @@ if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hera" ]]
   cd ..
 
   cd cpld_gridgen
-  export ACCOUNT=$PROJECT_CODE
-  export STMP=$WORK_DIR/reg-tests
 
   ./rt.sh
 
@@ -160,7 +176,7 @@ echo "Commit hash: ${current_hash}" >> ${WORK_DIR}/reg_test_results.txt
 echo "" >> ${WORK_DIR}/reg_test_results.txt
 
 success=true
-for dir in weight_gen ocnice_prep cpld_gridgen chgres_cube grid_gen global_cycle ice_blend snow2mdl; do
+for dir in regrid_sfc weight_gen ocnice_prep cpld_gridgen chgres_cube grid_gen global_cycle ice_blend snow2mdl; do
     if grep -qi "FAILED" ${dir}/summary.log; then
         success=false
         echo "${dir} consistency tests FAILED" >> ${WORK_DIR}/reg_test_results.txt

sorc/CMakeLists.txt

@@ -16,6 +16,9 @@ if(GCYCLE)
     message(STATUS "OpenMP is required for global_cycle.fd and was NOT found, skipping ...")
   endif()
 endif()
+if(REGRID)
+  add_subdirectory(regrid_sfc.fd)
+endif()
 if(FRENCTOOLS)
   add_subdirectory(fre-nctools.fd)
 endif()

sorc/regrid_sfc.fd/CMakeLists.txt

@@ -0,0 +1,46 @@
+# This is the CMake file for the regrid_sfc utility in the UFS_UTILS
+# project.
+#
+# George Gayno, Clara Draper
+
+set(lib_src
+   grids_IO.F90
+   readin_setup.F90
+   utils.F90)
+
+set(exe_src  regridStates.F90)
+
+# coppied from global_cycle 
+if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel|IntelLLVM)$")
+  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8")
+elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$")
+  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -ffree-line-length-0")
+endif()
+
+set(exe_name regridStates.x)
+
+add_library(regrid_lib STATIC ${lib_src})
+add_executable(${exe_name} ${exe_src})
+
+set(mod_dir "${CMAKE_CURRENT_BINARY_DIR}/mod")
+set_target_properties(regrid_lib PROPERTIES Fortran_MODULE_DIRECTORY ${mod_dir})
+target_include_directories(regrid_lib INTERFACE ${mod_dir})
+target_link_libraries(
+  regrid_lib
+  PUBLIC
+  esmf
+  MPI::MPI_Fortran
+  NetCDF::NetCDF_Fortran)
+
+if(OpenMP_Fortran_FOUND)
+  target_link_libraries(regrid_lib PUBLIC OpenMP::OpenMP_Fortran)
+endif()
+
+target_link_libraries(${exe_name} PRIVATE regrid_lib)
+
+install(TARGETS ${exe_name})
+
+# If doxygen documentation we enabled, build it.
+if(ENABLE_DOCS)
+  add_subdirectory(docs)  
+endif()