Update module_write_netcdf to avoid hangs in RRFS runs

tests/ci/repo_check.sh

@@ -1,111 +1,112 @@
 #!/bin/bash
 set -eu
 
-# This script checks if head repo of PR is up to date with ufs-weather-model develop
-# Checks for top level (ufs-weather-model) and next level components (submodules)
-result() {
-  if [[ -n $comment ]]; then
-    logID=$1
-    comment="@$logID please bring these up to date with respective authoritative repositories\n"$comment
-    printf %s "$comment"
-    #exit 1
-  fi
+get_shas () {
+    cwd=$(pwd)
+    # Get sha-1's of the top of develop and feature branches
+    app="Accept: application/vnd.github.v3+json"
+    url=$1
+    gitapi=$2
+    branch=$3
+    base_sha=$(curl -sS -H "$app" $gitapi | jq -r '.commit.sha')
+    workspace=$4
+    cd $workspace
+    git remote add upstream $url
+    git fetch -q upstream $branch
+    common=$(git merge-base $base_sha @)
+    echo $common $base_sha $workspace
+    if [[ $common != $base_sha ]]; then
+        printf "%s\n\n" "** $workspace **NOT** up to date"
+        flag_sync=false
+    fi
+    cd $cwd
 }
 
-# Declare variables
-declare -A base fv3 mom6 cice ww3 stoch gocart cmeps cdeps hycom cmake ccpp-framework ccpp-physics atmos_cubed_sphere
-submodules="fv3 mom6 cice ww3 stoch gocart cmeps cdeps hycom cmake ccpp-framework ccpp-physics atmos_cubed_sphere"
-comment=''
+flag_sync=true
+
 ownerID=$1
 
-# Base branch: this is the top of develop of ufs-weather-model
-base[repo]='https://github.com/ufs-community/ufs-weather-model'
-base[branch]='develop'
+declare -A urls branches pathes
+submodules="base fv3 mom6 cice ww3 stoch gocart cmeps cdeps hycom cmake ccpp_physics ccpp_framework aqm noahmp"
 
-# Submodules to check
-fv3[repo]='https://github.com/NOAA-EMC/fv3atm'
-fv3[branch]='develop'
-fv3[dir]='FV3'
+urls[base]='https://github.com/ufs-community/ufs-weather-model'
+branches[base]='develop'
+pathes[base]=''
 
-mom6[repo]='https://github.com/NOAA-EMC/MOM6'
-mom6[branch]='dev/emc'
-mom6[dir]='MOM6-interface/MOM6'
+urls[fv3]='https://github.com/NOAA-EMC/fv3atm'
+branches[fv3]='develop'
+pathes[fv3]='FV3'
 
-cice[repo]='https://github.com/NOAA-EMC/CICE'
-cice[branch]='emc/develop'
-cice[dir]='CICE-interface/CICE'
+urls[mom6]='https://github.com/NOAA-EMC/MOM6'
+branches[mom6]='dev/emc'
+pathes[mom6]='MOM6-interface/MOM6'
 
-ww3[repo]='https://github.com/NOAA-EMC/WW3'
-ww3[branch]='dev/ufs-weather-model'
-ww3[dir]='WW3'
+urls[cice]='https://github.com/NOAA-EMC/CICE'
+branches[cice]='emc/develop'
+pathes[cice]='CICE-interface/CICE'
 
-stoch[repo]='https://github.com/noaa-psl/stochastic_physics'
-stoch[branch]='master'
-stoch[dir]='stochastic_physics'
+urls[ww3]='https://github.com/NOAA-EMC/WW3'
+branches[ww3]='dev/ufs-weather-model'
+pathes[ww3]='WW3'
 
-gocart[repo]='https://github.com/GEOS-ESM/GOCART'
-gocart[branch]='develop'
-gocart[dir]='GOCART'
+urls[stoch]='https://github.com/noaa-psl/stochastic_physics'
+branches[stoch]='master'
+pathes[stoch]='stochastic_physics'
 
-cmeps[repo]='https://github.com/NOAA-EMC/CMEPS'
-cmeps[branch]='emc/develop'
-cmeps[dir]='CMEPS-interface/CMEPS'
+urls[gocart]='https://github.com/GEOS-ESM/GOCART'
+branches[gocart]='develop'
+pathes[gocart]='GOCART'
 
-cdeps[repo]='https://github.com/NOAA-EMC/CDEPS'
-cdeps[branch]='develop'
-cdeps[dir]='CDEPS-interface/CDEPS'
+urls[cmeps]='https://github.com/NOAA-EMC/CMEPS'
+branches[cmeps]='emc/develop'
+pathes[cmeps]='CMEPS-interface/CMEPS'
 
-hycom[repo]='https://github.com/NOAA-EMC/HYCOM-src'
-hycom[branch]='emc/develop'
-hycom[dir]='HYCOM-interface/HYCOM'
+urls[cdeps]='https://github.com/NOAA-EMC/CDEPS'
+branches[cdeps]='develop'
+pathes[cdeps]='CDEPS-interface/CDEPS'
 
-cmake[repo]='https://github.com/NOAA-EMC/CMakeModules'
-cmake[branch]='develop'
-cmake[dir]='CMakeModules'
+urls[hycom]='https://github.com/NOAA-EMC/HYCOM-src'
+branches[hycom]='emc/develop'
+pathes[hycom]='HYCOM-interface/HYCOM'
 
-ccpp-framework[repo]='https://github.com/NCAR/ccpp-framework'
-ccpp-framework[branch]='main'
-ccpp-framework[dir]='FV3/ccpp/framework'
+urls[cmake]='https://github.com/NOAA-EMC/CMakeModules'
+branches[cmake]='develop'
+pathes[cmake]='CMakeModules'
 
-ccpp-physics[repo]='https://github.com/ufs-community/ccpp-physics'
-ccpp-physics[branch]='ufs/dev'
-ccpp-physics[dir]='FV3/ccpp/physics'
+urls[ccpp_physics]='https://github.com/ufs-community/ccpp-physics'
+branches[ccpp_physics]='ufs/dev'
+pathes[ccpp_physics]='FV3/ccpp/physics'
 
-#upp[repo]='https://github.com/NOAA-EMC/UPP'
-#upp[branch]='develop'
-#upp[dir]='upp'
+urls[ccpp_framework]='https://github.com/NCAR/ccpp-framework'
+branches[ccpp_framework]='main'
+pathes[ccpp_framework]='FV3/ccpp/framework'
 
-atmos_cubed_sphere[repo]='https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere'
-atmos_cubed_sphere[branch]='dev/emc'
-atmos_cubed_sphere[dir]='FV3/atmos_cubed_sphere'
+urls[aqm]='https://github.com/NOAA-EMC/AQM'
+branches[aqm]='develop'
+pathes[aqm]='AQM'
 
-# Get sha-1's of the top of develop of ufs-weather-model
-app="Accept: application/vnd.github.v3+json"
-url="https://api.github.com/repos/ufs-community/ufs-weather-model/branches/develop"
-base[sha]=$(curl -sS -H "$app" $url | jq -r '.commit.sha')
-for submodule in $submodules; do
-  eval url=https://api.github.com/repos/ufs-community/ufs-weather-model/contents/'${'$submodule'[dir]}'
-  eval $submodule'[sha]=$(curl -sS -H "$app" $url | jq -r '.sha')'
-done
+urls[noahmp]='https://github.com/NOAA-EMC/noahmp'
+branches[noahmp]='develop'
+pathes[noahmp]='NOAHMP-interface/noahmp'
 
-# Check if the head branch is up to date with the base branch
-cd ${GITHUB_WORKSPACE}
-git remote add upstream ${base[repo]}
-git fetch -q upstream ${base[branch]}
-common=$(git merge-base ${base[sha]} @)
-if [[ $common != ${base[sha]} ]]; then
-  comment="* ufs-weather-model **NOT** up to date\n"
-fi
+#urls[upp]='https://github.com/NOAA-EMC/UPP'
+#branches[upp]='develop'
+#pathes[upp]='upp'
+
+#urls[cubed_sphere]='https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere'
+#branches[cubed_sphere]='dev/emc'
+#pathes[cubed_sphere]='FV3/atmos_cubed_sphere'
 
 for submodule in $submodules; do
-  eval cd ${GITHUB_WORKSPACE}/'${'$submodule'[dir]}'
-  eval git remote add upstream '${'$submodule'[repo]}'
-  eval git fetch -q upstream '${'$submodule'[branch]}'
-  common=$(eval git merge-base '${'$submodule'[sha]}' @)
-  if (eval test $common != '${'$submodule'[sha]}'); then
-    comment+="* $submodule **NOT** up to date\n"
-  fi
+    url=${urls[$submodule]}
+    branch=${branches[$submodule]}
+    workspace=${GITHUB_WORKSPACE}'/'${pathes[$submodule]}
+    gitapi=$(echo "$url" | sed 's/github.com/api.github.com\/repos/g')'/branches/'$branch
+    get_shas $url $gitapi $branch $workspace
 done
 
-result $ownerID
-exit 0
+if [[ $flag_sync=='true' ]]; then
+    exit 0
+else
+    exit 0
+fi

tests/logs/OpnReqTests_control_p8_hera.log

@@ -1,9 +1,9 @@
-Thu Mar 28 15:57:29 UTC 2024
+Mon Apr  1 12:58:41 UTC 2024
 Start Operation Requirement Test
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_bit_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_1301478/bit_base_bit_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_3809779/bit_base_bit_base
 Checking test bit_base results ....
 Moving baseline bit_base files ....
  Moving sfcf000.nc .........OK
@@ -51,14 +51,14 @@ Moving baseline bit_base files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 272.712165
-  0: The maximum resident set size (KB)                   = 1267880
+  0: The total amount of wall time                        = 268.251032
+  0: The maximum resident set size (KB)                   = 1265156
 
 Test bit_base PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_dbg_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_1301478/dbg_base_dbg_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_3809779/dbg_base_dbg_base
 Checking test dbg_base results ....
 Moving baseline dbg_base files ....
  Moving sfcf000.nc .........OK
@@ -106,14 +106,14 @@ Moving baseline dbg_base files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 976.383245
-  0: The maximum resident set size (KB)                   = 1259420
+  0: The total amount of wall time                        = 965.834768
+  0: The maximum resident set size (KB)                   = 1256504
 
 Test dbg_base PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_1301478/dcp_dcp
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_3809779/dcp_dcp
 Checking test dcp results ....
  Comparing sfcf000.nc .....USING NCCMP......OK
  Comparing sfcf021.nc .....USING NCCMP......OK
@@ -160,14 +160,14 @@ Checking test dcp results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .....USING NCCMP......OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .....USING NCCMP......OK
 
-  0: The total amount of wall time                        = 244.611359
-  0: The maximum resident set size (KB)                   = 1246672
+  0: The total amount of wall time                        = 246.586805
+  0: The maximum resident set size (KB)                   = 1233548
 
 Test dcp PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_1301478/mpi_mpi
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_3809779/mpi_mpi
 Checking test mpi results ....
  Comparing sfcf000.nc .....USING NCCMP......OK
  Comparing sfcf021.nc .....USING NCCMP......OK
@@ -214,14 +214,14 @@ Checking test mpi results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .....USING NCCMP......OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .....USING NCCMP......OK
 
-  0: The total amount of wall time                        = 240.307501
-  0: The maximum resident set size (KB)                   = 1240192
+  0: The total amount of wall time                        = 235.970450
+  0: The maximum resident set size (KB)                   = 1247740
 
 Test mpi PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_1301478/rst_rst
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_3809779/rst_rst
 Checking test rst results ....
  Comparing sfcf000.nc .....USING NCCMP......OK
  Comparing sfcf021.nc .....USING NCCMP......OK
@@ -268,14 +268,14 @@ Checking test rst results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .....USING NCCMP......OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .....USING NCCMP......OK
 
-  0: The total amount of wall time                        = 244.559360
-  0: The maximum resident set size (KB)                   = 1249316
+  0: The total amount of wall time                        = 242.245564
+  0: The maximum resident set size (KB)                   = 1242900
 
 Test rst PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_1301478/std_base_std_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_3809779/std_base_std_base
 Checking test std_base results ....
 Moving baseline std_base files ....
  Moving sfcf000.nc .........OK
@@ -323,14 +323,14 @@ Moving baseline std_base files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 241.325911
-  0: The maximum resident set size (KB)                   = 1245732
+  0: The total amount of wall time                        = 237.390294
+  0: The maximum resident set size (KB)                   = 1248812
 
 Test std_base PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_1301478/thr_thr
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_3809779/thr_thr
 Checking test thr results ....
  Comparing sfcf000.nc .....USING NCCMP......OK
  Comparing sfcf021.nc .....USING NCCMP......OK
@@ -377,11 +377,11 @@ Checking test thr results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .....USING NCCMP......OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .....USING NCCMP......OK
 
-  0: The total amount of wall time                        = 243.431160
-  0: The maximum resident set size (KB)                   = 1246648
+  0: The total amount of wall time                        = 242.382720
+  0: The maximum resident set size (KB)                   = 1242900
 
 Test thr PASS
 
 OPERATION REQUIREMENT TEST WAS SUCCESSFUL
-Thu Mar 28 17:55:09 UTC 2024
-Elapsed time: 01h:57m:40s. Have a nice day!
+Mon Apr  1 14:07:52 UTC 2024
+Elapsed time: 01h:09m:14s. Have a nice day!

tests/logs/OpnReqTests_cpld_control_nowave_noaero_p8_hera.log

@@ -1,9 +1,9 @@
-Fri Mar 29 15:59:03 UTC 2024
+Mon Apr  1 18:08:14 UTC 2024
 Start Operation Requirement Test
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/cpld_control_c96_noaero_p8_dbg_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_1890279/dbg_base_dbg_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_4186917/dbg_base_dbg_base
 Checking test dbg_base results ....
 Moving baseline dbg_base files ....
  Moving sfcf021.tile1.nc .........OK
@@ -66,14 +66,14 @@ Moving baseline dbg_base files ....
  Moving RESTART/iced.2021-03-23-21600.nc .........OK
  Moving RESTART/ufs.cpld.cpl.r.2021-03-23-21600.nc .........OK
 
-  0: The total amount of wall time                        = 1521.977133
-  0: The maximum resident set size (KB)                   = 1364132
+  0: The total amount of wall time                        = 1299.447405
+  0: The maximum resident set size (KB)                   = 1367532
 
 Test dbg_base PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/cpld_control_c96_noaero_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_1890279/rst_rst
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_4186917/rst_rst
 Checking test rst results ....
  Comparing sfcf021.tile1.nc .....USING NCCMP......OK
  Comparing sfcf021.tile2.nc .....USING NCCMP......OK
@@ -135,14 +135,14 @@ Checking test rst results ....
  Comparing RESTART/iced.2021-03-23-21600.nc .....USING NCCMP......OK
  Comparing RESTART/ufs.cpld.cpl.r.2021-03-23-21600.nc .....USING NCCMP......OK
 
-  0: The total amount of wall time                        = 377.430537
-  0: The maximum resident set size (KB)                   = 1358388
+  0: The total amount of wall time                        = 604.635134
+  0: The maximum resident set size (KB)                   = 1359688
 
 Test rst PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/cpld_control_c96_noaero_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_1890279/std_base_std_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_4186917/std_base_std_base
 Checking test std_base results ....
 Moving baseline std_base files ....
  Moving sfcf021.tile1.nc .........OK
@@ -205,11 +205,11 @@ Moving baseline std_base files ....
  Moving RESTART/iced.2021-03-23-21600.nc .........OK
  Moving RESTART/ufs.cpld.cpl.r.2021-03-23-21600.nc .........OK
 
-  0: The total amount of wall time                        = 672.561875
-  0: The maximum resident set size (KB)                   = 1359300
+  0: The total amount of wall time                        = 385.824022
+  0: The maximum resident set size (KB)                   = 1360152
 
 Test std_base PASS
 
 OPERATION REQUIREMENT TEST WAS SUCCESSFUL
-Fri Mar 29 17:09:20 UTC 2024
-Elapsed time: 01h:10m:18s. Have a nice day!
+Mon Apr  1 19:12:24 UTC 2024
+Elapsed time: 01h:04m:10s. Have a nice day!