Updates for Gaea C5 & F2/F5 Transition + Fix out of bound errors in block_atmos_copy routines in fv3atm #2124

modulefiles/ufs_gaea-c5.intel.lua → modulefiles/ufs_gaea.intel.lua

@@ -10,8 +10,8 @@ load("intel-classic/2023.1.0")
 load("cray-mpich/8.1.25")
 load("python/3.9.12")
 
-prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/modulefiles")
+prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/modulefiles")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
 load(pathJoin("stack-intel", stack_intel_ver))

tests/ci/Dockerfile

@@ -13,7 +13,7 @@ ENV test_name=$test_name
 ENV run_case=$run_case
 ENV CI_TEST=true
 ENV RT_COMPILER=gnu
-ENV RT_MACHINE=linux
+ENV MACHINE=linux
 ENV MACHINE_ID=linux
 # ACCNR CAN BE ANYTHING; DOCKER DOES NOT NEED THIS
 ENV ACCNR=nems 

tests/compile.sh

@@ -58,8 +58,8 @@ if [[ $MACHINE_ID == macosx ]] || [[ $MACHINE_ID == linux ]]; then
   source $PATHTR/modulefiles/ufs_${MACHINE_ID}.${RT_COMPILER}
 else
   # Activate lua environment for gaea c5
-  if [[ $MACHINE_ID == gaea-c5 ]]; then
-    source /lustre/f2/dev/role.epic/contrib/Lmod_init_C5.sh
+  if [[ $MACHINE_ID == gaea ]]; then
+    module reset
   fi
   # Load fv3 module
   module use $PATHTR/modulefiles

tests/default_vars.sh

@@ -216,7 +216,7 @@ elif [[ $MACHINE_ID = s4 ]]; then
   ICE_tasks_cpl_bmrk=48
   WAV_tasks_cpl_bmrk=80
 
-elif [[ $MACHINE_ID = gaea-c5 ]]; then
+elif [[ $MACHINE_ID = gaea ]]; then
 
   TPN=128
 

tests/detect_machine.sh

@@ -1,73 +1,41 @@
 #!/bin/bash
 
-# Do not set ACCNR here or it will break the per-system defaults in rt.sh.
-
+# The authoritative copy of this script lives in the ufs-weather-model at:
+# https://github.com/ufs-community/ufs-weather-model/blob/develop/tests/detect_machine.sh
+# If any local modifications are made or new platform support added,
+# please consider opening an issue and a PR to the ufs-weather-model
+# so that this copy remains in sync with its authoritative source
+#
+# Thank you for your contribution
+
+# If the MACHINE_ID variable is set, skip this script.
+[[ -n ${MACHINE_ID:-} ]] && return
+
+# First detect w/ hostname
 case $(hostname -f) in
 
-  clogin01.cactus.wcoss2.ncep.noaa.gov)	  MACHINE_ID=wcoss2 ;; ### cactus
-  clogin02.cactus.wcoss2.ncep.noaa.gov)	  MACHINE_ID=wcoss2 ;; ### cactus
-  clogin03.cactus.wcoss2.ncep.noaa.gov)	  MACHINE_ID=wcoss2 ;; ### cactus
-  clogin04.cactus.wcoss2.ncep.noaa.gov)	  MACHINE_ID=wcoss2 ;; ### cactus
-  clogin05.cactus.wcoss2.ncep.noaa.gov)	  MACHINE_ID=wcoss2 ;; ### cactus
-  clogin06.cactus.wcoss2.ncep.noaa.gov)	  MACHINE_ID=wcoss2 ;; ### cactus
-  clogin07.cactus.wcoss2.ncep.noaa.gov)	  MACHINE_ID=wcoss2 ;; ### cactus
-  clogin08.cactus.wcoss2.ncep.noaa.gov)	  MACHINE_ID=wcoss2 ;; ### cactus
-  clogin09.cactus.wcoss2.ncep.noaa.gov)	  MACHINE_ID=wcoss2 ;; ### cactus
-
-  dlogin01.dogwood.wcoss2.ncep.noaa.gov)  MACHINE_ID=wcoss2 ;; ### dogwood
-  dlogin02.dogwood.wcoss2.ncep.noaa.gov)  MACHINE_ID=wcoss2 ;; ### dogwood
-  dlogin03.dogwood.wcoss2.ncep.noaa.gov)  MACHINE_ID=wcoss2 ;; ### dogwood
-  dlogin04.dogwood.wcoss2.ncep.noaa.gov)  MACHINE_ID=wcoss2 ;; ### dogwood
-  dlogin05.dogwood.wcoss2.ncep.noaa.gov)  MACHINE_ID=wcoss2 ;; ### dogwood
-  dlogin06.dogwood.wcoss2.ncep.noaa.gov)  MACHINE_ID=wcoss2 ;; ### dogwood
-  dlogin07.dogwood.wcoss2.ncep.noaa.gov)  MACHINE_ID=wcoss2 ;; ### dogwood
-  dlogin08.dogwood.wcoss2.ncep.noaa.gov)  MACHINE_ID=wcoss2 ;; ### dogwood
-  dlogin09.dogwood.wcoss2.ncep.noaa.gov)  MACHINE_ID=wcoss2 ;; ### dogwood
-
-  alogin01.acorn.wcoss2.ncep.noaa.gov)  MACHINE_ID=acorn ;; ### acorn
-  alogin02.acorn.wcoss2.ncep.noaa.gov)  MACHINE_ID=acorn ;; ### acorn
-  alogin03.acorn.wcoss2.ncep.noaa.gov)  MACHINE_ID=acorn ;; ### acorn
-
-  gaea51.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea51
-  gaea52.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea52
-  gaea53.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea53
-  gaea54.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea54
-  gaea55.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea55
-  gaea56.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea56
-  gaea57.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea57
-  gaea58.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea58
-
-  hfe01)                   MACHINE_ID=hera ;; ### hera01
-  hfe02)                   MACHINE_ID=hera ;; ### hera02
-  hfe03)                   MACHINE_ID=hera ;; ### hera03
-  hfe04)                   MACHINE_ID=hera ;; ### hera04
-  hfe05)                   MACHINE_ID=hera ;; ### hera05
-  hfe06)                   MACHINE_ID=hera ;; ### hera06
-  hfe07)                   MACHINE_ID=hera ;; ### hera07
-  hfe08)                   MACHINE_ID=hera ;; ### hera08
-  hfe09)                   MACHINE_ID=hera ;; ### hera09
-  hfe10)                   MACHINE_ID=hera ;; ### hera10
-  hfe11)                   MACHINE_ID=hera ;; ### hera11
-  hfe12)                   MACHINE_ID=hera ;; ### hera12
-  hecflow01)               MACHINE_ID=hera ;; ### heraecflow01
+  adecflow0[12].acorn.wcoss2.ncep.noaa.gov)  MACHINE_ID=acorn ;; ### acorn
+  alogin0[123].acorn.wcoss2.ncep.noaa.gov)    MACHINE_ID=acorn ;; ### acorn
+  clogin0[1-9].cactus.wcoss2.ncep.noaa.gov)  MACHINE_ID=wcoss2 ;; ### cactus01-9
+  clogin10.cactus.wcoss2.ncep.noaa.gov)      MACHINE_ID=wcoss2 ;; ### cactus10
+  dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9
+  dlogin10.dogwood.wcoss2.ncep.noaa.gov)     MACHINE_ID=wcoss2 ;; ### dogwood10
+
+  gaea5[1-8])          MACHINE_ID=gaea ;; ### gaea51-58
+  gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58
+
+  hfe0[1-9]) MACHINE_ID=hera ;; ### hera01-09
+  hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12
+  hecflow01) MACHINE_ID=hera ;; ### heraecflow01
 
   s4-submit.ssec.wisc.edu) MACHINE_ID=s4 ;; ### s4
 
-  fe1)                     MACHINE_ID=jet ;; ### jet01
-  fe2)                     MACHINE_ID=jet ;; ### jet02
-  fe3)                     MACHINE_ID=jet ;; ### jet03
-  fe4)                     MACHINE_ID=jet ;; ### jet04
-  fe5)                     MACHINE_ID=jet ;; ### jet05
-  fe6)                     MACHINE_ID=jet ;; ### jet06
-  fe7)                     MACHINE_ID=jet ;; ### jet07
-  fe8)                     MACHINE_ID=jet ;; ### jet08
-  tfe1)                    MACHINE_ID=jet ;; ### jet09
-  tfe2)                    MACHINE_ID=jet ;; ### jet10
-
-  Orion-login-1.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1
-  Orion-login-2.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion2
-  Orion-login-3.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion3
-  Orion-login-4.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion4
+  fe[1-8]) MACHINE_ID=jet ;; ### jet01-8
+  tfe[12]) MACHINE_ID=jet ;; ### tjet1-2
+
+  Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4
+
+  [Hh]ercules-login-[1-4].[Hh][Pp][Cc].[Mm]s[Ss]tate.[Ee]du) MACHINE_ID=hercules ;; ### hercules1-4
 
   derecho1.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho1
   derecho2.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho2
@@ -77,31 +45,57 @@ case $(hostname -f) in
   derecho6.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho6
   derecho7.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho7
   derecho8.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho8
-
-  Hercules-login-1.HPC.MsState.Edu) MACHINE_ID=hercules;; ### hercules1 
-  Hercules-login-2.HPC.MsState.Edu) MACHINE_ID=hercules;; ### hercules2 
-  Hercules-login-3.HPC.MsState.Edu) MACHINE_ID=hercules;; ### hercules3
-  Hercules-login-4.HPC.MsState.Edu) MACHINE_ID=hercules;; ### hercules4
-
-  login1.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1
-  login2.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede2
-  login3.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede3
-  login4.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede4
-
-
-  login01.expanse.sdsc.edu) MACHINE_ID=expanse ;; ### expanse1
-  login02.expanse.sdsc.edu) MACHINE_ID=expanse ;; ### expanse2
 
-esac
+  login[1-4].stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1-4
 
-case $(echo ${PW_CSP:-nono}) in
+  login0[1-2].expanse.sdsc.edu) MACHINE_ID=expanse ;; ### expanse1-2
 
-  aws) MACHINE_ID=aws ;; ### parallelworks aws
-  google)  MACHINE_ID=gcp ;; ### parallelworks gcp
-  azure)  MACHINE_ID=azure ;; ### parallelworks azure
-
+  discover3[1-5].prv.cube) MACHINE_ID=discover ;; ### discover31-35
+  *) MACHINE_ID=UNKNOWN ;;  # Unknown platform
 esac
-[[ ${MACHINE_ID} =~ "aws" || ${MACHINE_ID} =~ "gcp" || ${MACHINE_ID} =~ "azure" ]] && MACHINE_ID=noaacloud
 
-# Overwrite auto-detect with RT_MACHINE if set
-MACHINE_ID=${RT_MACHINE:-${MACHINE_ID}}
+if [[ ${MACHINE_ID} == "UNKNOWN" ]]; then 
+   case ${PW_CSP:-} in
+      "aws" | "google" | "azure") MACHINE_ID=noaacloud ;;
+      *) PW_CSP="UNKNOWN"
+   esac
+fi
+
+# Overwrite auto-detect with MACHINE if set
+MACHINE_ID=${MACHINE:-${MACHINE_ID}}
+
+# If MACHINE_ID is no longer UNKNNOWN, return it
+if [[ "${MACHINE_ID}" != "UNKNOWN" ]]; then
+  return
+fi
+
+# Try searching based on paths since hostname may not match on compute nodes
+if [[ -d /lfs/h3 ]]; then
+  # We are on NOAA Cactus or Dogwood
+  MACHINE_ID=wcoss2
+elif [[ -d /lfs/h1 && ! -d /lfs/h3 ]]; then
+  # We are on NOAA TDS Acorn
+  MACHINE_ID=acorn
+elif [[ -d /mnt/lfs1 ]]; then
+  # We are on NOAA Jet
+  MACHINE_ID=jet
+elif [[ -d /scratch1 ]]; then
+  # We are on NOAA Hera
+  MACHINE_ID=hera
+elif [[ -d /work ]]; then
+  # We are on MSU Orion or Hercules
+  if [[ -d /apps/other ]]; then
+    # We are on Hercules
+    MACHINE_ID=hercules
+  else
+    MACHINE_ID=orion
+  fi
+elif [[ -d /gpfs && -d /ncrc ]]; then
+  # We are on GAEA.
+  MACHINE_ID=gaea
+elif [[ -d /data/prod ]]; then
+  # We are on SSEC's S4
+  MACHINE_ID=s4
+else
+  echo WARNING: UNKNOWN PLATFORM 1>&2
+fi

tests/logs/OpnReqTests_control_p8_hera.log

@@ -1,9 +1,9 @@
-Thu Feb  8 14:00:04 UTC 2024
+Fri Feb  9 14:39:17 UTC 2024
 Start Operation Requirement Test
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_bit_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_103025/control_p8_gnu_bit_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_204164/control_p8_gnu_bit_base
 Checking test bit_base control_p8_gnu results ....
 Moving baseline bit_base control_p8_gnu files ....
  Moving sfcf000.nc .........OK
@@ -51,14 +51,14 @@ Moving baseline bit_base control_p8_gnu files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 277.080729
-  0: The maximum resident set size (KB)                   = 1311728
+  0: The total amount of wall time                        = 282.839267
+  0: The maximum resident set size (KB)                   = 1304660
 
 Test bit_base control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_dbg_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_103025/control_p8_gnu_dbg_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_204164/control_p8_gnu_dbg_base
 Checking test dbg_base control_p8_gnu results ....
 Moving baseline dbg_base control_p8_gnu files ....
  Moving sfcf000.nc .........OK
@@ -106,14 +106,14 @@ Moving baseline dbg_base control_p8_gnu files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 903.668548
-  0: The maximum resident set size (KB)                   = 1287728
+  0: The total amount of wall time                        = 892.021338
+  0: The maximum resident set size (KB)                   = 1288444
 
 Test dbg_base control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_103025/control_p8_gnu_dcp
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_204164/control_p8_gnu_dcp
 Checking test dcp control_p8_gnu results ....
  Comparing sfcf000.nc .........OK
  Comparing sfcf021.nc .........OK
@@ -160,14 +160,14 @@ Checking test dcp control_p8_gnu results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 249.952287
-  0: The maximum resident set size (KB)                   = 1281684
+  0: The total amount of wall time                        = 249.299884
+  0: The maximum resident set size (KB)                   = 1281236
 
 Test dcp control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_103025/control_p8_gnu_mpi
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_204164/control_p8_gnu_mpi
 Checking test mpi control_p8_gnu results ....
  Comparing sfcf000.nc .........OK
  Comparing sfcf021.nc .........OK
@@ -214,14 +214,14 @@ Checking test mpi control_p8_gnu results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 251.057241
-  0: The maximum resident set size (KB)                   = 1283180
+  0: The total amount of wall time                        = 247.586878
+  0: The maximum resident set size (KB)                   = 1284604
 
 Test mpi control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_103025/control_p8_gnu_rst
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_204164/control_p8_gnu_rst
 Checking test rst control_p8_gnu results ....
  Comparing sfcf000.nc .........OK
  Comparing sfcf021.nc .........OK
@@ -268,14 +268,14 @@ Checking test rst control_p8_gnu results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 247.649190
-  0: The maximum resident set size (KB)                   = 1283604
+  0: The total amount of wall time                        = 247.664516
+  0: The maximum resident set size (KB)                   = 1283488
 
 Test rst control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_103025/control_p8_gnu_std_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_204164/control_p8_gnu_std_base
 Checking test std_base control_p8_gnu results ....
 Moving baseline std_base control_p8_gnu files ....
  Moving sfcf000.nc .........OK
@@ -323,14 +323,14 @@ Moving baseline std_base control_p8_gnu files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 247.317862
-  0: The maximum resident set size (KB)                   = 1286884
+  0: The total amount of wall time                        = 249.494655
+  0: The maximum resident set size (KB)                   = 1281868
 
 Test std_base control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_103025/control_p8_gnu_thr
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_204164/control_p8_gnu_thr
 Checking test thr control_p8_gnu results ....
  Comparing sfcf000.nc .........OK
  Comparing sfcf021.nc .........OK
@@ -377,11 +377,11 @@ Checking test thr control_p8_gnu results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 249.513148
-  0: The maximum resident set size (KB)                   = 1282944
+  0: The total amount of wall time                        = 249.472280
+  0: The maximum resident set size (KB)                   = 1283528
 
 Test thr control_p8_gnu PASS
 
 OPERATION REQUIREMENT TEST WAS SUCCESSFUL
-Thu Feb  8 15:16:58 UTC 2024
-Elapsed time: 01h:16m:54s. Have a nice day!
+Fri Feb  9 17:11:49 UTC 2024
+Elapsed time: 02h:32m:32s. Have a nice day!