From 88c18813a4f6a879f2d3249c937214783cbd827a Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Wed, 4 Jun 2025 20:27:16 +0000 Subject: [PATCH 01/54] Add build module for ursa. Update source code to access required modules for ip v5. Fixes #1064. --- modulefiles/build.ursa.intelllvm.lua | 62 ++++++++++++++++++++++++ sorc/chgres_cube.fd/model_grid.F90 | 1 + sorc/global_cycle.fd/cycle.f90 | 1 + sorc/global_cycle.fd/land_increments.f90 | 1 + sorc/machine-setup.sh | 17 +++++-- sorc/weight_gen.fd/scrip.F90 | 2 + 6 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 modulefiles/build.ursa.intelllvm.lua diff --git a/modulefiles/build.ursa.intelllvm.lua b/modulefiles/build.ursa.intelllvm.lua new file mode 100644 index 000000000..8df2bde4c --- /dev/null +++ b/modulefiles/build.ursa.intelllvm.lua @@ -0,0 +1,62 @@ +help([[ +Load environment to compile UFS_UTILS on Ursa using Intelllvm +]]) + +prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.2.1/install/modulefiles/Core") + +stack_intel_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" +load(pathJoin("stack-oneapi", stack_oneapi_ver)) + +stack_oneapi_mpi_ver=os.getenv("stack_oneapi_mpi_ver") or "2021.13" +load(pathJoin("stack-intel-oneapi-mpi", stack_oneapi_mpi_ver)) + +cmake_ver=os.getenv("cmake_ver") or "3.27.9" +load(pathJoin("cmake", cmake_ver)) + +bacio_ver=os.getenv("bacio_ver") or "2.4.1" +load(pathJoin("bacio", bacio_ver)) + +g2_ver=os.getenv("g2_ver") or "3.5.1" +load(pathJoin("g2", g2_ver)) + +ip_ver=os.getenv("ip_ver") or "5.1.0" +load(pathJoin("ip", ip_ver)) + +nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" +load(pathJoin("nemsio", nemsio_ver)) + +--sp_ver=os.getenv("sp_ver") or "2.5.0" +--load(pathJoin("sp", sp_ver)) + +w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" +load(pathJoin("w3emc", w3emc_ver)) + +-- Uncomment when CHGRES_ALL is ON +--sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" +--load(pathJoin("sfcio", sfcio_ver)) + +sigio_ver=os.getenv("sigio_ver") or "2.3.3" +load(pathJoin("sigio", sigio_ver)) + +netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" +load(pathJoin("netcdf-c", netcdf_c_ver)) + +netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1" +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) + +nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" +load(pathJoin("nccmp", nccmp_ver)) + +esmf_ver=os.getenv("esmf_ver") or "8.6.1" +load(pathJoin("esmf", esmf_ver)) + +nco_ver=os.getenv("nco_ver") or "5.2.4" +load(pathJoin("nco", nco_ver)) + +setenv("I_MPI_CC", "icx") +setenv("I_MPI_FC", "ifx") + +setenv("CC", "mpicc") +setenv("FC", "mpifc") + +whatis("Description: UFS_UTILS build environment") diff --git a/sorc/chgres_cube.fd/model_grid.F90 b/sorc/chgres_cube.fd/model_grid.F90 index 74f4c341d..3c8f0c407 100644 --- a/sorc/chgres_cube.fd/model_grid.F90 +++ b/sorc/chgres_cube.fd/model_grid.F90 @@ -161,6 +161,7 @@ subroutine define_input_grid_gaussian(npets) use sfcio_module use sigio_module #endif + use sp_mod use netcdf implicit none diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index 6431dca62..b3e85ba08 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -924,6 +924,7 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& USE UTILS USE GDSWZD_MOD + use sp_mod USE READ_WRITE_DATA, ONLY : IDIM_GAUS, JDIM_GAUS, & SLMSK_GAUS, DTREF_GAUS, & NSST_DATA diff --git a/sorc/global_cycle.fd/land_increments.f90 b/sorc/global_cycle.fd/land_increments.f90 index 0d7297b59..b02c5e790 100644 --- a/sorc/global_cycle.fd/land_increments.f90 +++ b/sorc/global_cycle.fd/land_increments.f90 @@ -48,6 +48,7 @@ subroutine gaussian_to_fv3_interp(lsoil_incr,rla,rlo, & use utils use gdswzd_mod + use sp_mod use read_write_data, only : idim_gaus, jdim_gaus, & stc_inc_gaus, soilsnow_gaus, slc_inc_gaus use mpi diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index 1765b5922..52dba04bc 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -46,6 +46,14 @@ elif [[ -d /scratch1 ]] ; then fi target=hera module purge +elif [[ -d /scratch3 ]] ; then + # We are on NOAA Ursa + if ( ! eval module help > /dev/null 2>&1 ) ; then + echo load the module command 1>&2 + source /apps/lmod/lmod/init/$__ms_shell + fi + target=ursa + module purge elif [[ "$(hostname)" == "gaea5"* || "$(hostname)" =~ c5n[0-9]+ ]] && [[ -d /gpfs/f5 ]] ; then # We are on GAEAC5. if ( ! eval module help > /dev/null 2>&1 ) ; then @@ -80,15 +88,18 @@ elif [[ -d /data/prod ]] ; then module purge else if [[ ! -v PW_CSP ]]; then - echo WARNING: UNKNOWN PLATFORM 1>&2; exit 99 + set +x + echo FATAL ERROR: UNKNOWN PLATFORM 1>&2; exit 99 elif [[ -z "${PW_CSP}" ]]; then - echo WARNING: UNKNOWN PLATFORM 1>&2; exit 99 + set +x + echo FATAL ERROR: UNKNOWN PLATFORM 1>&2; exit 99 else if [[ "${PW_CSP}" == "aws" || "${PW_CSP}" == "azure" || "${PW_CSP}" == "google" ]]; then target=noaacloud module purge else - echo WARNING: UNKNOWN PLATFORM 1>&2; exit 99 + set +x + echo FATAL ERROR: UNKNOWN PLATFORM 1>&2; exit 99 fi fi fi diff --git a/sorc/weight_gen.fd/scrip.F90 b/sorc/weight_gen.fd/scrip.F90 index 839767dfc..c20afaba5 100644 --- a/sorc/weight_gen.fd/scrip.F90 +++ b/sorc/weight_gen.fd/scrip.F90 @@ -18,6 +18,8 @@ !! @author George Gayno NOAA/EMC program scrip + use sp_mod + implicit none character(len=128) :: outfile From bcd5a321803676c45c5920da32fe93ceaab271be Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 5 Jun 2025 12:01:22 +0000 Subject: [PATCH 02/54] Remove references to Hera from PR template. Fixes #1064. --- .github/PULL_REQUEST_TEMPLATE | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 717dac75b..a99b830df 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -16,8 +16,8 @@ One or more paragraphs describing the problem, solution, and required changes. ## TESTS CONDUCTED: If there are changes to the build or source code, the tests below must be conducted. Contact a repository manager if you need assistance. -- [ ] Compile branch on all Tier 1 machines using Intel (Orion, Jet, Hera, Hercules and WCOSS2). -- [ ] Compile branch on Hera using GNU. +- [ ] Compile branch on all Tier 1 machines using Intel (Orion, Jet, Ursa, Hercules and WCOSS2). +- [ ] Compile branch on Ursa using GNU. - [ ] Compile branch in 'Debug' mode on WCOSS2. - [ ] Compile with Doxygen on any machine with no errors. - [ ] Run unit tests locally on any Tier 1 machine. @@ -25,7 +25,7 @@ If there are changes to the build or source code, the tests below must be conduc Optional test. -- [ ] Run full set of chgres_cube consistency tests on Hera. +- [ ] Run full set of chgres_cube consistency tests on Ursa. Describe any additional tests performed. From 8c8e2089ee1e9675d69db22343d46af671a4cb3d Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 5 Jun 2025 12:52:36 +0000 Subject: [PATCH 03/54] Add pre-processing directives so code can compile with pre-V5 versions of IPLIB. Fixes #1064. --- sorc/chgres_cube.fd/CMakeLists.txt | 4 ++++ sorc/chgres_cube.fd/model_grid.F90 | 2 ++ sorc/global_cycle.fd/CMakeLists.txt | 8 ++++++-- sorc/global_cycle.fd/{cycle.f90 => cycle.F90} | 2 ++ .../{land_increments.f90 => land_increments.F90} | 2 ++ sorc/weight_gen.fd/CMakeLists.txt | 3 +++ sorc/weight_gen.fd/scrip.F90 | 2 ++ 7 files changed, 21 insertions(+), 2 deletions(-) rename sorc/global_cycle.fd/{cycle.f90 => cycle.F90} (99%) rename sorc/global_cycle.fd/{land_increments.f90 => land_increments.F90} (99%) diff --git a/sorc/chgres_cube.fd/CMakeLists.txt b/sorc/chgres_cube.fd/CMakeLists.txt index 4ff86450f..70d7dc13c 100644 --- a/sorc/chgres_cube.fd/CMakeLists.txt +++ b/sorc/chgres_cube.fd/CMakeLists.txt @@ -31,6 +31,10 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ffree-line-length-0 -fdefault-real-8") endif() +if(ip_VERSION GREATER_EQUAL 5.0.0) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DIP_V5") +endif() + if(CHGRES_ALL) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DCHGRES_ALL") endif() diff --git a/sorc/chgres_cube.fd/model_grid.F90 b/sorc/chgres_cube.fd/model_grid.F90 index 3c8f0c407..60d79479b 100644 --- a/sorc/chgres_cube.fd/model_grid.F90 +++ b/sorc/chgres_cube.fd/model_grid.F90 @@ -161,7 +161,9 @@ subroutine define_input_grid_gaussian(npets) use sfcio_module use sigio_module #endif +#ifdef IP_V5 use sp_mod +#endif use netcdf implicit none diff --git a/sorc/global_cycle.fd/CMakeLists.txt b/sorc/global_cycle.fd/CMakeLists.txt index f8120f00a..7a2da4ed1 100644 --- a/sorc/global_cycle.fd/CMakeLists.txt +++ b/sorc/global_cycle.fd/CMakeLists.txt @@ -9,9 +9,9 @@ set(lib_src ../../ccpp-physics/physics/Interstitials/UFS_SCM_NEPTUNE/sfcsub.F read_write_data.f90 utils.F90 - land_increments.f90) + land_increments.F90) -set(exe_src cycle.f90) +set(exe_src cycle.F90) if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel|IntelLLVM)$") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8") @@ -22,6 +22,10 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$") endif() endif() +if(ip_VERSION GREATER_EQUAL 5.0.0) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DIP_V5") +endif() + set(exe_name global_cycle) add_library(global_cycle_lib STATIC ${lib_src}) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.F90 similarity index 99% rename from sorc/global_cycle.fd/cycle.f90 rename to sorc/global_cycle.fd/cycle.F90 index b3e85ba08..493888d7d 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.F90 @@ -924,7 +924,9 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& USE UTILS USE GDSWZD_MOD +#ifdef IP_V5 use sp_mod +#endif USE READ_WRITE_DATA, ONLY : IDIM_GAUS, JDIM_GAUS, & SLMSK_GAUS, DTREF_GAUS, & NSST_DATA diff --git a/sorc/global_cycle.fd/land_increments.f90 b/sorc/global_cycle.fd/land_increments.F90 similarity index 99% rename from sorc/global_cycle.fd/land_increments.f90 rename to sorc/global_cycle.fd/land_increments.F90 index b02c5e790..6efae21a8 100644 --- a/sorc/global_cycle.fd/land_increments.f90 +++ b/sorc/global_cycle.fd/land_increments.F90 @@ -48,7 +48,9 @@ subroutine gaussian_to_fv3_interp(lsoil_incr,rla,rlo, & use utils use gdswzd_mod +#ifdef IP_V5 use sp_mod +#endif use read_write_data, only : idim_gaus, jdim_gaus, & stc_inc_gaus, soilsnow_gaus, slc_inc_gaus use mpi diff --git a/sorc/weight_gen.fd/CMakeLists.txt b/sorc/weight_gen.fd/CMakeLists.txt index 4c408d441..3890aa6d7 100644 --- a/sorc/weight_gen.fd/CMakeLists.txt +++ b/sorc/weight_gen.fd/CMakeLists.txt @@ -7,6 +7,9 @@ if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel|IntelLLVM)$") elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8") endif() +if(ip_VERSION GREATER_EQUAL 5.0.0) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DIP_V5") +endif() set(exe_name weight_gen) add_executable(${exe_name} ${fortran_src}) diff --git a/sorc/weight_gen.fd/scrip.F90 b/sorc/weight_gen.fd/scrip.F90 index c20afaba5..0ff0ac3a7 100644 --- a/sorc/weight_gen.fd/scrip.F90 +++ b/sorc/weight_gen.fd/scrip.F90 @@ -18,7 +18,9 @@ !! @author George Gayno NOAA/EMC program scrip +#ifdef IP_V5 use sp_mod +#endif implicit none From 2d2e498520c9c0e987cb5803b29a4729a150c05d Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 5 Jun 2025 13:32:25 +0000 Subject: [PATCH 04/54] Port weight_gen regression test. Fixes #1064. --- reg_tests/weight_gen/{driver.hera.sh => driver.ursa.sh} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename reg_tests/weight_gen/{driver.hera.sh => driver.ursa.sh} (89%) diff --git a/reg_tests/weight_gen/driver.hera.sh b/reg_tests/weight_gen/driver.ursa.sh similarity index 89% rename from reg_tests/weight_gen/driver.hera.sh rename to reg_tests/weight_gen/driver.ursa.sh index d88011cc7..a57cb3788 100755 --- a/reg_tests/weight_gen/driver.hera.sh +++ b/reg_tests/weight_gen/driver.ursa.sh @@ -2,7 +2,7 @@ #----------------------------------------------------------------------------- # -# Run weight_gen consistency test on Hera. +# Run weight_gen consistency test on Ursa. # # Set $DATA to your working directory. Set the project code (SBATCH -A) # and queue (SBATCH -q) as appropriate. @@ -36,7 +36,7 @@ module use ../../modulefiles module load build.$target.$compiler module list -export DATA="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}" +export DATA="${WORK_DIR:-/scratch4/NCEPDEV/stmp/$LOGNAME}" export DATA="${DATA}/reg-tests/weight_gen" #----------------------------------------------------------------------------- @@ -50,7 +50,7 @@ if [ "$UPDATE_BASELINE" = "TRUE" ]; then source ../get_hash.sh fi -export HOMEreg=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/weight_gen +export HOMEreg=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/weight_gen export HOMEufs=$PWD/../.. ./weight_gen.sh From c87540b2b25a1c4d8e0030d67658414606ea38c9 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 5 Jun 2025 15:54:54 +0000 Subject: [PATCH 05/54] Update regrid_sfc regression test driver script. Fixes #1064. --- reg_tests/regrid_sfc/driver.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/reg_tests/regrid_sfc/driver.sh b/reg_tests/regrid_sfc/driver.sh index 822a4729f..6889cff97 100755 --- a/reg_tests/regrid_sfc/driver.sh +++ b/reg_tests/regrid_sfc/driver.sh @@ -53,28 +53,29 @@ if [[ "$target" == "jet" ]];then QUEUE="${QUEUE:-batch}" export HOMEreg=/lfs5/HFIP/hfv3gfs/emc.nemspara/role.ufsutils/ufs_utils/reg_tests/regrid_sfc export APRUN_REGRID=srun - PARTITION=xjet -elif [[ "$target" == "hera" ]];then - WORK_DIR="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}" + PARTITION="--partition=xjet" +elif [[ "$target" == "ursa" ]];then + WORK_DIR="${WORK_DIR:-/scratch4/NCEPDEV/stmp/$LOGNAME}" PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}" QUEUE="${QUEUE:-batch}" - export HOMEreg=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/regrid_sfc/ +# export HOMEreg=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/regrid_sfc/ + export HOMEreg=/scratch3/NCEPDEV/da/George.Gayno/noscrub/reg_tests/regrid_sfc/ export APRUN_REGRID=srun - PARTITION=hera + PARTITION='' elif [[ "$target" == "orion" ]];then WORK_DIR="${WORK_DIR:-/work/noaa/stmp/$LOGNAME}" PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}" QUEUE="${QUEUE:-batch}" export HOMEreg=/work/noaa/nems/role-nems/ufs_utils/reg_tests/regrid_sfc export APRUN_REGRID=srun - PARTITION=orion + PARTITION='' elif [[ "$target" == "hercules" ]];then WORK_DIR="${WORK_DIR:-/work2/noaa/stmp/$LOGNAME}" PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}" QUEUE="${QUEUE:-batch}" export HOMEreg=/work/noaa/nems/role-nems/ufs_utils.hercules/reg_tests/regrid_sfc export APRUN_REGRID=srun - PARTITION=hercules + PARTITION='' elif [[ "$target" == "wcoss2" ]];then WORK_DIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" PROJECT_CODE="${PROJECT_CODE:-GFS-DEV}" @@ -94,7 +95,7 @@ if [[ "$target" == "wcoss2" ]];then -N gauss2fv3incr -l select=1:ncpus=6:ompthreads=1:mem=10GB ./gauss2fv3incr.sh) else TEST1=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J gauss2fv3incr \ - --partition=$PARTITION -o $LOG_FILE -e $LOG_FILE ./gauss2fv3incr.sh) + $PARTITION -o $LOG_FILE -e $LOG_FILE ./gauss2fv3incr.sh) fi LOG_FILE=consistency.log @@ -112,8 +113,8 @@ EOF else -sbatch --partition=$PARTITION --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J summary -o $LOG_FILE -e $LOG_FILE \ - --open-mode=append -q $QUEUE -d afterok:$TEST1 << EOF +sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J summary -o $LOG_FILE -e $LOG_FILE \ + $PARTITION --open-mode=append -q $QUEUE -d afterok:$TEST1 << EOF #!/bin/bash grep -a '<<<' ${LOG_FILE}* > ./summary.log EOF From a217459c6844b3a57c8e0c741eb354d936694e92 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 5 Jun 2025 16:01:55 +0000 Subject: [PATCH 06/54] Update ./fix/link_fixdirs.sh for ursa. Remove references to hera. Fixes #1064. --- fix/link_fixdirs.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fix/link_fixdirs.sh b/fix/link_fixdirs.sh index f14b03080..14b30809e 100755 --- a/fix/link_fixdirs.sh +++ b/fix/link_fixdirs.sh @@ -9,7 +9,7 @@ set -ex # 'nco' (copies data). # # $machine - is the machine. Choices are: -# 'wcoss2', 'hera', 'jet', 'orion', 'hercules', 's4', 'gaeac5', 'gaeac6' +# 'wcoss2', 'ursa', 'jet', 'orion', 'hercules', 's4', 'gaeac5', 'gaeac6' RUN_ENVIR=${1} machine=${2} @@ -17,7 +17,7 @@ machine=${2} if [ $# -lt 2 ]; then set +x echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine' - echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | hera | jet | orion | hercules | s4 | gaeac5 | gaeac6 )' + echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | ursa | jet | orion | hercules | s4 | gaeac5 | gaeac6 )' exit 1 fi @@ -28,10 +28,10 @@ if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then exit 1 fi -if [ $machine != wcoss2 -a $machine != hera -a $machine != jet -a $machine != orion -a $machine != s4 -a $machine != hercules -a $machine != gaeac5 -a $machine != gaeac6 ]; then +if [ $machine != wcoss2 -a $machine != ursa -a $machine != jet -a $machine != orion -a $machine != s4 -a $machine != hercules -a $machine != gaeac5 -a $machine != gaeac6 ]; then set +x echo '***ERROR*** unsupported machine' - echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | hera | jet | orion | hercules | s4 | gaeac5 | gaeac6 )' + echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | ursa | jet | orion | hercules | s4 | gaeac5 | gaeac6 )' exit 1 fi @@ -44,8 +44,8 @@ pwd=$(pwd -P) #------------------------------ #--model fix fields #------------------------------ -if [ $machine = "hera" ]; then - FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix" +if [ $machine = "ursa" ]; then + FIX_DIR="/scratch3/NCEPDEV/global/role.glopara/fix" elif [ $machine = "jet" ]; then FIX_DIR="/lfs5/HFIP/hfv3gfs/glopara/FIX/fix" elif [ $machine = "orion" -o $machine = "hercules" ]; then From 7942d4f5d875e9a44e204bfa3520b2456f3983bb Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 5 Jun 2025 17:06:46 +0000 Subject: [PATCH 07/54] Port ice blend regression test script to ursa. Fixes #1064. --- .../ice_blend/{driver.hera.sh => driver.ursa.sh} | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) rename reg_tests/ice_blend/{driver.hera.sh => driver.ursa.sh} (76%) diff --git a/reg_tests/ice_blend/driver.hera.sh b/reg_tests/ice_blend/driver.ursa.sh similarity index 76% rename from reg_tests/ice_blend/driver.hera.sh rename to reg_tests/ice_blend/driver.ursa.sh index 7a6fb4c86..50fc8651d 100755 --- a/reg_tests/ice_blend/driver.hera.sh +++ b/reg_tests/ice_blend/driver.ursa.sh @@ -2,7 +2,7 @@ #----------------------------------------------------------------------------- # -# Run ice_blend consistency test on Hera. +# Run ice_blend consistency test on Ursa. # # Set $DATA to your working directory. Set the project code (SBATCH -A) # and queue (SBATCH -q) as appropriate. @@ -35,12 +35,14 @@ source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.$compiler module load grib-util -# Because of a bug in the grib-util module, the wgrib2 module -# must be loaded last. -module load wgrib2/2.0.8 +module load wgrib2/3.6.0 module list -export DATA="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}" +export CNVGRIB=${GRIB_UTIL_ROOT}/bin/cnvgrib +export COPYGB=${GRIB_UTIL_ROOT}/bin/copygb +export COPYGB2=${GRIB_UTIL_ROOT}/bin/copygb2 + +export DATA="${WORK_DIR:-/scratch4/NCEPDEV/stmp/$LOGNAME}" export DATA="${DATA}/reg-tests/ice-blend" #----------------------------------------------------------------------------- @@ -54,7 +56,8 @@ if [ "$UPDATE_BASELINE" = "TRUE" ]; then source ../get_hash.sh fi -export HOMEreg=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/ice_blend +#export HOMEreg=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/ice_blend +export HOMEreg=/scratch3/NCEPDEV/da/George.Gayno/noscrub/reg_tests/ice_blend export HOMEgfs=$PWD/../.. rm -fr $DATA From 0dc6260454c3275f993d6aa52f99b1cff5c84c3d Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 5 Jun 2025 18:42:38 +0000 Subject: [PATCH 08/54] Port snow2mdl regression test. Fixes #1064. --- .../{driver.hera.sh => driver.ursa.sh} | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) rename reg_tests/snow2mdl/{driver.hera.sh => driver.ursa.sh} (80%) diff --git a/reg_tests/snow2mdl/driver.hera.sh b/reg_tests/snow2mdl/driver.ursa.sh similarity index 80% rename from reg_tests/snow2mdl/driver.hera.sh rename to reg_tests/snow2mdl/driver.ursa.sh index 03d66c75e..fecfe39cf 100755 --- a/reg_tests/snow2mdl/driver.hera.sh +++ b/reg_tests/snow2mdl/driver.ursa.sh @@ -2,7 +2,7 @@ #----------------------------------------------------------------------------- # -# Run snow2mdl consistency tests on Hera. +# Run snow2mdl consistency tests on Ursa. # # Set $DATA_ROOT to your working directory. Set the project code (SBATCH -A) # and queue (SBATCH -q) as appropriate. @@ -26,18 +26,16 @@ source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.$compiler module load grib-util -module load wgrib2/2.0.8 -module load prod_util/2.1.1 +module load wgrib2 +module load prod_util module list -# Because of a bug in the grib-util module, need to construct this -# variable. -WGRIB=${grib_util_ROOT}/bin/wgrib +WGRIB=${GRIB_UTIL_ROOT}/bin/wgrib export WGRIB export WGRIB2 -DATA_ROOT="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}" +DATA_ROOT="${WORK_DIR:-/scratch4/NCEPDEV/stmp/$LOGNAME}" DATA_ROOT="${DATA_ROOT}/reg-tests/snow2mdl" rm -fr $DATA_ROOT @@ -56,20 +54,21 @@ if [ "$UPDATE_BASELINE" = "TRUE" ]; then source ../get_hash.sh fi -export HOMEreg=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/snow2mdl +#export HOMEreg=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/snow2mdl +export HOMEreg=/scratch3/NCEPDEV/da/George.Gayno/noscrub/reg_tests/snow2mdl export HOMEgfs=$PWD/../.. # The first test uses hemispheric afwa/airforce data, as was done in OPS. export DATA="${DATA_ROOT}/test.hemi" TEST1=$(sbatch --parsable -J snow.hemi -A ${PROJECT_CODE} -o consistency.log -e consistency.log \ - --ntasks=1 -q ${QUEUE} -t 00:03:00 ./snow2mdl.hemi.sh) + --ntasks=1 --mem=5GB -q ${QUEUE} -t 00:03:00 ./snow2mdl.hemi.sh) # The second test mimics current OPS, which uses global afwa/airforce data. export DATA="${DATA_ROOT}/test.global" TEST2=$(sbatch --parsable -J snow.global -A ${PROJECT_CODE} -o consistency.log -e consistency.log \ - --ntasks=1 -q ${QUEUE} -t 00:03:00 -d afterok:$TEST1 ./snow2mdl.global.sh) + --ntasks=1 --mem=5GB -q ${QUEUE} -t 00:03:00 -d afterok:$TEST1 ./snow2mdl.global.sh) # Create summary file. From 97657bc8dc11db937cfcefde8c66aa80ac4ba07c Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 5 Jun 2025 19:38:35 +0000 Subject: [PATCH 09/54] Point to official baseline directories. Fixes #1064. --- reg_tests/ice_blend/driver.ursa.sh | 3 +-- reg_tests/regrid_sfc/driver.sh | 3 +-- reg_tests/snow2mdl/driver.ursa.sh | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/reg_tests/ice_blend/driver.ursa.sh b/reg_tests/ice_blend/driver.ursa.sh index 50fc8651d..0bf72da87 100755 --- a/reg_tests/ice_blend/driver.ursa.sh +++ b/reg_tests/ice_blend/driver.ursa.sh @@ -56,8 +56,7 @@ if [ "$UPDATE_BASELINE" = "TRUE" ]; then source ../get_hash.sh fi -#export HOMEreg=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/ice_blend -export HOMEreg=/scratch3/NCEPDEV/da/George.Gayno/noscrub/reg_tests/ice_blend +export HOMEreg=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/ice_blend export HOMEgfs=$PWD/../.. rm -fr $DATA diff --git a/reg_tests/regrid_sfc/driver.sh b/reg_tests/regrid_sfc/driver.sh index 6889cff97..babc44dfa 100755 --- a/reg_tests/regrid_sfc/driver.sh +++ b/reg_tests/regrid_sfc/driver.sh @@ -58,8 +58,7 @@ elif [[ "$target" == "ursa" ]];then WORK_DIR="${WORK_DIR:-/scratch4/NCEPDEV/stmp/$LOGNAME}" PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}" QUEUE="${QUEUE:-batch}" -# export HOMEreg=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/regrid_sfc/ - export HOMEreg=/scratch3/NCEPDEV/da/George.Gayno/noscrub/reg_tests/regrid_sfc/ + export HOMEreg=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/regrid_sfc/ export APRUN_REGRID=srun PARTITION='' elif [[ "$target" == "orion" ]];then diff --git a/reg_tests/snow2mdl/driver.ursa.sh b/reg_tests/snow2mdl/driver.ursa.sh index fecfe39cf..d91f9380b 100755 --- a/reg_tests/snow2mdl/driver.ursa.sh +++ b/reg_tests/snow2mdl/driver.ursa.sh @@ -54,8 +54,7 @@ if [ "$UPDATE_BASELINE" = "TRUE" ]; then source ../get_hash.sh fi -#export HOMEreg=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/snow2mdl -export HOMEreg=/scratch3/NCEPDEV/da/George.Gayno/noscrub/reg_tests/snow2mdl +export HOMEreg=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/snow2mdl export HOMEgfs=$PWD/../.. # The first test uses hemispheric afwa/airforce data, as was done in OPS. From 921ce27cdc6a000aecc76d3efab8b90ec1d4c494 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 5 Jun 2025 20:11:58 +0000 Subject: [PATCH 10/54] Update ocnice_prep regression test script. Fixes #1064. --- .../RegressionTests_hera.intel.log | 34 ------------------- reg_tests/ocnice_prep/rt.sh | 22 ++++++------ 2 files changed, 11 insertions(+), 45 deletions(-) delete mode 100644 reg_tests/ocnice_prep/RegressionTests_hera.intel.log diff --git a/reg_tests/ocnice_prep/RegressionTests_hera.intel.log b/reg_tests/ocnice_prep/RegressionTests_hera.intel.log deleted file mode 100644 index c82cf588c..000000000 --- a/reg_tests/ocnice_prep/RegressionTests_hera.intel.log +++ /dev/null @@ -1,34 +0,0 @@ -Mon Jun 10 11:47:37 UTC 2024 -Start Regression test - -Working dir = /scratch1/NCEPDEV/stmp4/Denise.Worthen/OCNICE_PREP/rt_237427/050_ocean -Baseline dir = /scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/ocnice_prep/baseline_data/050_ocean - -Checking test 050_ocean results .... -Comparing ocean.mx050.nc........OK - - -Working dir = /scratch1/NCEPDEV/stmp4/Denise.Worthen/OCNICE_PREP/rt_237427/050_ice -Baseline dir = /scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/ocnice_prep/baseline_data/050_ice - -Checking test 050_ice results .... -Comparing ice.mx050.nc........OK - - -Working dir = /scratch1/NCEPDEV/stmp4/Denise.Worthen/OCNICE_PREP/rt_237427/100_ocean -Baseline dir = /scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/ocnice_prep/baseline_data/100_ocean - -Checking test 100_ocean results .... -Comparing ocean.mx100.nc........OK - - -Working dir = /scratch1/NCEPDEV/stmp4/Denise.Worthen/OCNICE_PREP/rt_237427/100_ice -Baseline dir = /scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/ocnice_prep/baseline_data/100_ice - -Checking test 100_ice results .... -Comparing ice.mx100.nc........OK - - -REGRESSION TEST WAS SUCCESSFUL -Mon Jun 10 11:56:38 UTC 2024 -Elapsed time: 00h:11m:00s. Have a nice day! diff --git a/reg_tests/ocnice_prep/rt.sh b/reg_tests/ocnice_prep/rt.sh index 1115831d5..7406aad6a 100755 --- a/reg_tests/ocnice_prep/rt.sh +++ b/reg_tests/ocnice_prep/rt.sh @@ -66,16 +66,16 @@ if [[ $target = wcoss2 ]]; then QUEUE=${QUEUE:-dev} WLCLK=15 export NCCMP=nccmp -elif [[ $target = hera ]]; then - STMP=${STMP:-/scratch1/NCEPDEV/stmp4/$USER} - BASELINE_ROOT=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/ocnice_prep/baseline_data - WEIGHTS_ROOT=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data - INPUT_ROOT=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/ocnice_prep/input_data +elif [[ $target = ursa ]]; then + STMP=${STMP:-/scratch4/NCEPDEV/stmp/$USER} + BASELINE_ROOT=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/ocnice_prep/baseline_data + WEIGHTS_ROOT=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data + INPUT_ROOT=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/ocnice_prep/input_data ACCOUNT=${ACCOUNT:-fv3-cpu} QUEUE=${QUEUE:-batch} WLCLK=10 export NCCMP=nccmp - PARTITION=hera + PARTITION='' elif [[ $target = orion ]]; then STMP=${STMP:-/work/noaa/stmp/$USER} BASELINE_ROOT=/work/noaa/nems/role-nems/ufs_utils/reg_tests/ocnice_prep/baseline_data @@ -85,7 +85,7 @@ elif [[ $target = orion ]]; then QUEUE=${QUEUE:-batch} WLCLK=15 export NCCMP=nccmp - PARTITION=orion + PARTITION='' ulimit -s unlimited elif [[ $target = hercules ]]; then STMP=${STMP:-/work2/noaa/stmp/$USER} @@ -96,7 +96,7 @@ elif [[ $target = hercules ]]; then QUEUE=${QUEUE:-batch} WLCLK=10 export NCCMP=nccmp - PARTITION=hercules + PARTITION='' ulimit -s unlimited elif [[ $target = jet ]]; then STMP=${STMP:-/lfs5/HFIP/h-nems/$USER} @@ -107,7 +107,7 @@ elif [[ $target = jet ]]; then QUEUE=${QUEUE:-batch} WLCLK=10 export NCCMP=nccmp - PARTITION=xjet + PARTITION="--partition=xjet" ulimit -s unlimited fi @@ -234,7 +234,7 @@ while read -r line || [ "$line" ]; do else tests[$i]=$(sbatch --parsable --ntasks-per-node=1 --nodes=1 --mem=24g -t 00:${WLCLK}:00 -A $ACCOUNT -q $QUEUE -J $TEST_NAME \ - --partition=$PARTITION -o run_${TEST_NAME}.log -e run_${TEST_NAME}.log ./ocnice_prep.sh "$TEST_NAME") + $PARTITION -o run_${TEST_NAME}.log -e run_${TEST_NAME}.log ./ocnice_prep.sh "$TEST_NAME") fi @@ -255,7 +255,7 @@ if [[ $target = wcoss2 ]]; then else sbatch --ntasks=1 --mem=25m -t 0:01:00 -A $ACCOUNT -J summary -o /dev/null -e /dev/null \ - --partition=$PARTITION --open-mode=append -q $QUEUE -d afterok${all_tests} ./rt.summary.sh + $PARTITION --open-mode=append -q $QUEUE -d afterok${all_tests} ./rt.summary.sh fi From 4c4d97cd123f8648f0949896581c67f3b4f054da Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 6 Jun 2025 11:53:07 +0000 Subject: [PATCH 11/54] Update cpld_gridgen regression test script. Fixes #1064. --- .../RegressionTests_hera.intel.log | 150 ------------------ reg_tests/cpld_gridgen/rt.sh | 22 +-- 2 files changed, 11 insertions(+), 161 deletions(-) delete mode 100644 reg_tests/cpld_gridgen/RegressionTests_hera.intel.log diff --git a/reg_tests/cpld_gridgen/RegressionTests_hera.intel.log b/reg_tests/cpld_gridgen/RegressionTests_hera.intel.log deleted file mode 100644 index 316fa4295..000000000 --- a/reg_tests/cpld_gridgen/RegressionTests_hera.intel.log +++ /dev/null @@ -1,150 +0,0 @@ -Mon Apr 8 19:09:11 UTC 2024 -Start Regression test - -Working dir = /scratch1/NCEPDEV/stmp4/Denise.Worthen/CPLD_GRIDGEN/rt_1965110/025 -Baseline dir = /scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data/025 - -Checking test 025 results .... -Comparing Bu.mx025_SCRIP.nc........OK -Comparing C384.mx025.tile1.nc........OK -Comparing C384.mx025.tile2.nc........OK -Comparing C384.mx025.tile3.nc........OK -Comparing C384.mx025.tile4.nc........OK -Comparing C384.mx025.tile5.nc........OK -Comparing C384.mx025.tile6.nc........OK -Comparing Ct.mx025_SCRIP_land.nc........OK -Comparing Ct.mx025_SCRIP.nc........OK -Comparing Ct.mx025.to.C384.nc........OK -Comparing Cu.mx025_SCRIP.nc........OK -Comparing Cv.mx025_SCRIP.nc........OK -Comparing grid_cice_NEMS_mx025.nc........OK -Comparing kmtu_cice_NEMS_mx025.nc........OK -Comparing mesh.mx025.nc........OK -Comparing rect.0p25_SCRIP.nc........OK -Comparing rect.0p50_SCRIP.nc........OK -Comparing rect.1p00_SCRIP.nc........OK -Comparing rect.5p00_SCRIP.nc........OK -Comparing tripole.mx025.Bu.to.Ct.bilinear.nc........OK -Comparing tripole.mx025.Ct.to.Bu.bilinear.nc........OK -Comparing tripole.mx025.Ct.to.rect.0p25.bilinear.nc........OK -Comparing tripole.mx025.Ct.to.rect.0p25.conserve.nc........OK -Comparing tripole.mx025.Ct.to.rect.0p50.bilinear.nc........OK -Comparing tripole.mx025.Ct.to.rect.0p50.conserve.nc........OK -Comparing tripole.mx025.Ct.to.rect.1p00.bilinear.nc........OK -Comparing tripole.mx025.Ct.to.rect.1p00.conserve.nc........OK -Comparing tripole.mx025.Ct.to.rect.5p00.bilinear.nc........OK -Comparing tripole.mx025.Ct.to.rect.5p00.conserve.nc........OK -Comparing tripole.mx025.Cu.to.Ct.bilinear.nc........OK -Comparing tripole.mx025.Cv.to.Ct.bilinear.nc........OK -Comparing tripole.mx025.nc........OK - - -Working dir = /scratch1/NCEPDEV/stmp4/Denise.Worthen/CPLD_GRIDGEN/rt_1965110/050 -Baseline dir = /scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data/050 - -Checking test 050 results .... -Comparing Bu.mx050_SCRIP.nc........OK -Comparing C192.mx050.tile1.nc........OK -Comparing C192.mx050.tile2.nc........OK -Comparing C192.mx050.tile3.nc........OK -Comparing C192.mx050.tile4.nc........OK -Comparing C192.mx050.tile5.nc........OK -Comparing C192.mx050.tile6.nc........OK -Comparing Ct.mx025_SCRIP.nc........OK -Comparing Ct.mx050_SCRIP_land.nc........OK -Comparing Ct.mx050_SCRIP.nc........OK -Comparing Ct.mx050.to.C192.nc........OK -Comparing Cu.mx050_SCRIP.nc........OK -Comparing Cv.mx050_SCRIP.nc........OK -Comparing grid_cice_NEMS_mx050.nc........OK -Comparing kmtu_cice_NEMS_mx050.nc........OK -Comparing mesh.mx050.nc........OK -Comparing rect.0p50_SCRIP.nc........OK -Comparing rect.1p00_SCRIP.nc........OK -Comparing rect.5p00_SCRIP.nc........OK -Comparing tripole.mx025.Ct.to.mx050.Ct.bilinear.nc........OK -Comparing tripole.mx025.Ct.to.mx050.Ct.neareststod.nc........OK -Comparing tripole.mx050.Bu.to.Ct.bilinear.nc........OK -Comparing tripole.mx050.Ct.to.Bu.bilinear.nc........OK -Comparing tripole.mx050.Ct.to.rect.0p50.bilinear.nc........OK -Comparing tripole.mx050.Ct.to.rect.0p50.conserve.nc........OK -Comparing tripole.mx050.Ct.to.rect.1p00.bilinear.nc........OK -Comparing tripole.mx050.Ct.to.rect.1p00.conserve.nc........OK -Comparing tripole.mx050.Ct.to.rect.5p00.bilinear.nc........OK -Comparing tripole.mx050.Ct.to.rect.5p00.conserve.nc........OK -Comparing tripole.mx050.Cu.to.Ct.bilinear.nc........OK -Comparing tripole.mx050.Cv.to.Ct.bilinear.nc........OK -Comparing tripole.mx050.nc........OK - - -Working dir = /scratch1/NCEPDEV/stmp4/Denise.Worthen/CPLD_GRIDGEN/rt_1965110/100 -Baseline dir = /scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data/100 - -Checking test 100 results .... -Comparing Bu.mx100_SCRIP.nc........OK -Comparing C96.mx100.tile1.nc........OK -Comparing C96.mx100.tile2.nc........OK -Comparing C96.mx100.tile3.nc........OK -Comparing C96.mx100.tile4.nc........OK -Comparing C96.mx100.tile5.nc........OK -Comparing C96.mx100.tile6.nc........OK -Comparing Ct.mx025_SCRIP.nc........OK -Comparing Ct.mx100_SCRIP_land.nc........OK -Comparing Ct.mx100_SCRIP.nc........OK -Comparing Ct.mx100.to.C96.nc........OK -Comparing Cu.mx100_SCRIP.nc........OK -Comparing Cv.mx100_SCRIP.nc........OK -Comparing grid_cice_NEMS_mx100.nc........OK -Comparing kmtu_cice_NEMS_mx100.nc........OK -Comparing mesh.mx100.nc........OK -Comparing rect.1p00_SCRIP.nc........OK -Comparing rect.5p00_SCRIP.nc........OK -Comparing tripole.mx025.Ct.to.mx100.Ct.bilinear.nc........OK -Comparing tripole.mx025.Ct.to.mx100.Ct.neareststod.nc........OK -Comparing tripole.mx100.Bu.to.Ct.bilinear.nc........OK -Comparing tripole.mx100.Ct.to.Bu.bilinear.nc........OK -Comparing tripole.mx100.Ct.to.rect.1p00.bilinear.nc........OK -Comparing tripole.mx100.Ct.to.rect.1p00.conserve.nc........OK -Comparing tripole.mx100.Ct.to.rect.5p00.bilinear.nc........OK -Comparing tripole.mx100.Ct.to.rect.5p00.conserve.nc........OK -Comparing tripole.mx100.Cu.to.Ct.bilinear.nc........OK -Comparing tripole.mx100.Cv.to.Ct.bilinear.nc........OK -Comparing tripole.mx100.nc........OK -Comparing ufs.topo_edits_011818.nc........OK - - -Working dir = /scratch1/NCEPDEV/stmp4/Denise.Worthen/CPLD_GRIDGEN/rt_1965110/500 -Baseline dir = /scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data/500 - -Checking test 500 results .... -Comparing Bu.mx500_SCRIP.nc........OK -Comparing C48.mx500.tile1.nc........OK -Comparing C48.mx500.tile2.nc........OK -Comparing C48.mx500.tile3.nc........OK -Comparing C48.mx500.tile4.nc........OK -Comparing C48.mx500.tile5.nc........OK -Comparing C48.mx500.tile6.nc........OK -Comparing Ct.mx025_SCRIP.nc........OK -Comparing Ct.mx500_SCRIP_land.nc........OK -Comparing Ct.mx500_SCRIP.nc........OK -Comparing Ct.mx500.to.C48.nc........OK -Comparing Cu.mx500_SCRIP.nc........OK -Comparing Cv.mx500_SCRIP.nc........OK -Comparing grid_cice_NEMS_mx500.nc........OK -Comparing kmtu_cice_NEMS_mx500.nc........OK -Comparing mesh.mx500.nc........OK -Comparing rect.5p00_SCRIP.nc........OK -Comparing tripole.mx025.Ct.to.mx500.Ct.bilinear.nc........OK -Comparing tripole.mx025.Ct.to.mx500.Ct.neareststod.nc........OK -Comparing tripole.mx500.Bu.to.Ct.bilinear.nc........OK -Comparing tripole.mx500.Ct.to.Bu.bilinear.nc........OK -Comparing tripole.mx500.Ct.to.rect.5p00.bilinear.nc........OK -Comparing tripole.mx500.Ct.to.rect.5p00.conserve.nc........OK -Comparing tripole.mx500.Cu.to.Ct.bilinear.nc........OK -Comparing tripole.mx500.Cv.to.Ct.bilinear.nc........OK -Comparing tripole.mx500.nc........OK - - -REGRESSION TEST WAS SUCCESSFUL -Mon Apr 8 19:31:41 UTC 2024 -Elapsed time: 00h:23m:55s. Have a nice day! diff --git a/reg_tests/cpld_gridgen/rt.sh b/reg_tests/cpld_gridgen/rt.sh index 0ca97556a..9a693119e 100755 --- a/reg_tests/cpld_gridgen/rt.sh +++ b/reg_tests/cpld_gridgen/rt.sh @@ -52,15 +52,15 @@ MOM6_version=20250128 # Adjust STMP, ACCOUNT and QUEUE as needed. -if [[ $target = hera ]]; then - STMP=${STMP:-/scratch2/NCEPDEV/stmp1/$USER} +if [[ $target = ursa ]]; then + STMP=${STMP:-/scratch4/NCEPDEV/stmp/$USER} ACCOUNT=${ACCOUNT:-fv3-cpu} QUEUE=${QUEUE:-batch} WLCLK=40 - export MOM6_FIXDIR=/scratch1/NCEPDEV/global/glopara/fix/mom6/${MOM6_version} + export MOM6_FIXDIR=/scratch3/NCEPDEV/global/role.glopara/fix/mom6/${MOM6_version} export NCCMP=nccmp - BASELINE_ROOT=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data - PARTITION=hera + BASELINE_ROOT=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data + PARTITION='' elif [[ $target = orion ]]; then STMP=${STMP:-/work/noaa/stmp/$USER} ACCOUNT=${ACCOUNT:-fv3-cpu} @@ -69,7 +69,7 @@ elif [[ $target = orion ]]; then export MOM6_FIXDIR=/work/noaa/global/glopara/fix/mom6/${MOM6_version} export NCCMP=nccmp BASELINE_ROOT=/work/noaa/nems/role-nems/ufs_utils/reg_tests/cpld_gridgen/baseline_data - PARTITION=orion + PARTITION='' ulimit -s unlimited elif [[ $target = hercules ]]; then STMP=${STMP:-/work2/noaa/stmp/$USER} @@ -79,7 +79,7 @@ elif [[ $target = hercules ]]; then export MOM6_FIXDIR=/work/noaa/global/glopara/fix/mom6/${MOM6_version} BASELINE_ROOT=/work/noaa/nems/role-nems/ufs_utils.hercules/reg_tests/cpld_gridgen/baseline_data export NCCMP=nccmp - PARTITION=hercules + PARTITION='' ulimit -s unlimited elif [[ $target = jet ]]; then STMP=${STMP:-/lfs5/HFIP/emcda/$USER/stmp} @@ -89,7 +89,7 @@ elif [[ $target = jet ]]; then export MOM6_FIXDIR=/lfs5/HFIP/hfv3gfs/glopara/FIX/fix/mom6/${MOM6_version} BASELINE_ROOT=/lfs5/HFIP/hfv3gfs/emc.nemspara/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data export NCCMP=nccmp - PARTITION=xjet + PARTITION="--partition=xjet" ulimit -s unlimited elif [[ $target = wcoss2 ]]; then STMP=${STMP:-/lfs/h2/emc/stmp/$USER} @@ -215,8 +215,8 @@ while read -r line || [ "$line" ]; do -l walltime=00:${WLCLK}:00 -N $TEST_NAME -l select=1:ncpus=1:mem=12GB -v RESNAME=$TEST_NAME,ATMLIST="'$ATMLIST'" ./cpld_gridgen.sh) else - tests[$i]=$(sbatch --parsable --ntasks-per-node=1 --nodes=1 -t 00:${WLCLK}:00 -A $ACCOUNT -q $QUEUE -J $TEST_NAME \ - --partition=$PARTITION -o run_${TEST_NAME}.log -e run_${TEST_NAME}.log ./cpld_gridgen.sh "$TEST_NAME" "$ATMLIST") + tests[$i]=$(sbatch --parsable --ntasks-per-node=1 --nodes=1 --mem=12GB -t 00:${WLCLK}:00 -A $ACCOUNT -q $QUEUE -J $TEST_NAME \ + $PARTITION -o run_${TEST_NAME}.log -e run_${TEST_NAME}.log ./cpld_gridgen.sh "$TEST_NAME" "$ATMLIST") fi all_tests=${all_tests}":"${tests[$i]} @@ -237,7 +237,7 @@ if [[ $target = wcoss2 ]]; then else sbatch --nodes=1 -t 0:01:00 -A $ACCOUNT -J summary -o /dev/null -e /dev/null \ - --partition=$PARTITION --open-mode=append -q $QUEUE -d afterok${all_tests} ./rt.summary.sh + $PARTITION --open-mode=append -q $QUEUE -d afterok${all_tests} ./rt.summary.sh fi From f5deea2038e4ffd86839790aec29d0dbc7d8d2a5 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Mon, 9 Jun 2025 14:20:04 +0000 Subject: [PATCH 12/54] Update grid_gen regression test script. Fixes #1064. --- reg_tests/grid_gen/{driver.hera.sh => driver.ursa.sh} | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) rename reg_tests/grid_gen/{driver.hera.sh => driver.ursa.sh} (92%) diff --git a/reg_tests/grid_gen/driver.hera.sh b/reg_tests/grid_gen/driver.ursa.sh similarity index 92% rename from reg_tests/grid_gen/driver.hera.sh rename to reg_tests/grid_gen/driver.ursa.sh index 5449316bf..152bedbd6 100755 --- a/reg_tests/grid_gen/driver.hera.sh +++ b/reg_tests/grid_gen/driver.ursa.sh @@ -2,7 +2,7 @@ #----------------------------------------------------------------------------- # -# Run grid generation consistency tests on Hera. +# Run grid generation consistency tests on Ursa. # # Set WORK_DIR to your working directory. Set the PROJECT_CODE and QUEUE # as appropriate. To see which projects you are authorized to use, @@ -30,7 +30,7 @@ module list set -x -export WORK_DIR="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}" +export WORK_DIR="${WORK_DIR:-/scratch4/NCEPDEV/stmp/$LOGNAME}" export WORK_DIR="${WORK_DIR}/reg-tests/grid-gen" QUEUE="${QUEUE:-batch}" PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}" @@ -52,10 +52,9 @@ export home_dir=$PWD/../.. export APRUN=time export APRUN_SFC=srun export OMP_STACKSIZE=2048m -export HOMEreg=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/grid_gen/baseline_data +export HOMEreg=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/grid_gen/baseline_data ulimit -a -#ulimit -s unlimited rm -fr $WORK_DIR @@ -74,8 +73,8 @@ TEST1=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:15:00 -A $PROJECT_ #----------------------------------------------------------------------------- LOG_FILE2=${LOG_FILE}02 -TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.viirs.bnu \ - --partition=bigmem -o $LOG_FILE2 -e $LOG_FILE2 ./c96.viirs.bnu.sh) +TEST2=$(sbatch --parsable --ntasks-per-node=12 --nodes=2 --mem=300g -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.viirs.bnu \ + -o $LOG_FILE2 -e $LOG_FILE2 ./c96.viirs.bnu.sh) #----------------------------------------------------------------------------- # gfdl regional grid From 4ff6b89ca32ba296010230b92af132e992c5d9b7 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Mon, 9 Jun 2025 17:36:50 +0000 Subject: [PATCH 13/54] Update the chgres_cube regression test. Fixes #1064. --- .../{driver.hera.sh => driver.ursa.sh} | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) rename reg_tests/chgres_cube/{driver.hera.sh => driver.ursa.sh} (82%) diff --git a/reg_tests/chgres_cube/driver.hera.sh b/reg_tests/chgres_cube/driver.ursa.sh similarity index 82% rename from reg_tests/chgres_cube/driver.hera.sh rename to reg_tests/chgres_cube/driver.ursa.sh index df9a7c065..b93afed22 100755 --- a/reg_tests/chgres_cube/driver.hera.sh +++ b/reg_tests/chgres_cube/driver.ursa.sh @@ -2,7 +2,7 @@ #----------------------------------------------------------------------------- # -# Run the chgres_cube consistency tests on Hera. +# Run the chgres_cube consistency tests on Ursa. # # Set WORK_DIR to a general working location outside the UFS_UTILS directory. # The exact working directory (OUTDIR) will be WORK_DIR/reg_tests/chgres-cube. @@ -36,7 +36,7 @@ module use ../../modulefiles module load build.$target.$compiler module list -export OUTDIR="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}" +export OUTDIR="${WORK_DIR:-/scratch4/NCEPDEV/stmp/$LOGNAME}" export OUTDIR="${OUTDIR}/reg-tests/chgres-cube" PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}" @@ -60,7 +60,7 @@ fi export HOMEufs=$PWD/../.. -export HOMEreg=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/chgres_cube +export HOMEreg=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/chgres_cube rm -f consistency.log* summary*log @@ -76,7 +76,7 @@ rm -fr $OUTDIR LOG_FILE=consistency.log01 export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST1=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.restart \ +TEST1=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 --mem=50g -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.restart \ -o $LOG_FILE -e $LOG_FILE ./c96.fv3.restart.sh) #----------------------------------------------------------------------------- @@ -85,7 +85,7 @@ TEST1=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_C LOG_FILE=consistency.log02 export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST2=$(sbatch --parsable --ntasks-per-node=6 --nodes=2 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c192.fv3.history \ +TEST2=$(sbatch --parsable --ntasks-per-node=6 --nodes=2 --mem=100g -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c192.fv3.history \ -o $LOG_FILE -e $LOG_FILE ./c192.fv3.history.sh) #----------------------------------------------------------------------------- @@ -94,7 +94,7 @@ TEST2=$(sbatch --parsable --ntasks-per-node=6 --nodes=2 -t 0:15:00 -A $PROJECT_C LOG_FILE=consistency.log03 export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST3=$(sbatch --parsable --ntasks-per-node=12 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.netcdf \ +TEST3=$(sbatch --parsable --ntasks-per-node=12 --nodes=1 --mem=100g -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.netcdf \ -o $LOG_FILE -e $LOG_FILE ./c96.fv3.netcdf.sh) #----------------------------------------------------------------------------- @@ -103,7 +103,7 @@ TEST3=$(sbatch --parsable --ntasks-per-node=12 --nodes=1 -t 0:15:00 -A $PROJECT_ LOG_FILE=consistency.log04 export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST4=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c192.gfs.grib2 \ +TEST4=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 --mem=50g -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c192.gfs.grib2 \ -o $LOG_FILE -e $LOG_FILE ./c192.gfs.grib2.sh) #----------------------------------------------------------------------------- @@ -112,7 +112,7 @@ TEST4=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_C LOG_FILE=consistency.log05 export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST5=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 25km.conus.gfs.grib2.conus \ +TEST5=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 --mem=50g -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 25km.conus.gfs.grib2.conus \ -o $LOG_FILE -e $LOG_FILE ./25km.conus.gfs.grib2.sh) #----------------------------------------------------------------------------- @@ -121,7 +121,7 @@ TEST5=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_C LOG_FILE=consistency.log06 export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST6=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J 3km.conus.hrrr.gfssdf.grib2.conus \ +TEST6=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 --mem=100g -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J 3km.conus.hrrr.gfssdf.grib2.conus \ -o $LOG_FILE -e $LOG_FILE ./3km.conus.hrrr.gfssdf.grib2.sh) #----------------------------------------------------------------------------- @@ -130,7 +130,7 @@ TEST6=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:10:00 -A $PROJECT_C LOG_FILE=consistency.log07 export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST7=$(sbatch --parsable --ntasks-per-node=6 --nodes=2 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J 3km.conus.hrrr.newsfc.grib2.conus \ +TEST7=$(sbatch --parsable --ntasks-per-node=6 --nodes=2 --mem=100g -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J 3km.conus.hrrr.newsfc.grib2.conus \ -o $LOG_FILE -e $LOG_FILE ./3km.conus.hrrr.newsfc.grib2.sh) #----------------------------------------------------------------------------- @@ -139,7 +139,7 @@ TEST7=$(sbatch --parsable --ntasks-per-node=6 --nodes=2 -t 0:10:00 -A $PROJECT_C LOG_FILE=consistency.log08 export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST8=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 13km.conus.nam.grib2.conus \ +TEST8=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 --mem=50g -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 13km.conus.nam.grib2.conus \ -o $LOG_FILE -e $LOG_FILE ./13km.conus.nam.grib2.sh) #----------------------------------------------------------------------------- @@ -148,7 +148,7 @@ TEST8=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_C LOG_FILE=consistency.log09 export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST9=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 13km.conus.rap.grib2.conus \ +TEST9=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 --mem=100g -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 13km.conus.rap.grib2.conus \ -o $LOG_FILE -e $LOG_FILE ./13km.conus.rap.grib2.sh) #----------------------------------------------------------------------------- @@ -157,7 +157,7 @@ TEST9=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_C LOG_FILE=consistency.log10 export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST10=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 13km.na.gfs.ncei.grib2.conus \ +TEST10=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 --mem=100g -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 13km.na.gfs.ncei.grib2.conus \ -o $LOG_FILE -e $LOG_FILE ./13km.na.gfs.ncei.grib2.sh) #----------------------------------------------------------------------------- @@ -166,7 +166,7 @@ TEST10=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_ LOG_FILE=consistency.log11 export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST11=$(sbatch --parsable --ntasks-per-node=12 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.netcdf2wam \ +TEST11=$(sbatch --parsable --ntasks-per-node=12 --nodes=1 --mem=100g -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.netcdf2wam \ -o $LOG_FILE -e $LOG_FILE ./c96.fv3.netcdf2wam.sh) #----------------------------------------------------------------------------- @@ -175,7 +175,7 @@ TEST11=$(sbatch --parsable --ntasks-per-node=12 --nodes=1 -t 0:15:00 -A $PROJECT LOG_FILE=consistency.log12 export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST12=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 25km.conus.gfs.pbgrib2.conus \ +TEST12=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 --mem=100g -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 25km.conus.gfs.pbgrib2.conus \ -o $LOG_FILE -e $LOG_FILE ./25km.conus.gfs.pbgrib2.sh) #----------------------------------------------------------------------------- @@ -184,26 +184,23 @@ TEST12=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_ LOG_FILE=consistency.log13 export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST13=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c96.gefs.grib2 \ +TEST13=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 --mem=50g -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c96.gefs.grib2 \ -o $LOG_FILE -e $LOG_FILE ./c96.gefs.grib2.sh) - #----------------------------------------------------------------------------- # Initialize CONUS 13-KM USING RAP-SMOKE GRIB2 file WITH GSD PHYSICS . #----------------------------------------------------------------------------- LOG_FILE=consistency.log14 export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST14=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 13km.conus.rap-smoke.grib2.conus \ +TEST14=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 --mem=100g -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 13km.conus.rap-smoke.grib2.conus \ -o $LOG_FILE -e $LOG_FILE ./13km.conus.rap-smoke.grib2.sh) - - #----------------------------------------------------------------------------- # Create summary log. #----------------------------------------------------------------------------- LOG_FILE=consistency.log -sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \ +sbatch --ntasks=1 -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \ --open-mode=append -q $QUEUE -d\ afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5:$TEST6:$TEST7:$TEST8:$TEST9:$TEST10:$TEST11:$TEST12:$TEST13:$TEST14 << EOF #!/bin/bash From 3fa8d61d7b3f7f318d3479c436ef1497e42d58b7 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Mon, 9 Jun 2025 19:41:05 +0000 Subject: [PATCH 14/54] Update global_cycle regression test. Fixes #1064. --- reg_tests/global_cycle/{driver.hera.sh => driver.ursa.sh} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename reg_tests/global_cycle/{driver.hera.sh => driver.ursa.sh} (94%) diff --git a/reg_tests/global_cycle/driver.hera.sh b/reg_tests/global_cycle/driver.ursa.sh similarity index 94% rename from reg_tests/global_cycle/driver.hera.sh rename to reg_tests/global_cycle/driver.ursa.sh index 206a325f4..b8e5551d1 100755 --- a/reg_tests/global_cycle/driver.hera.sh +++ b/reg_tests/global_cycle/driver.ursa.sh @@ -2,7 +2,7 @@ #----------------------------------------------------------------------------- # -# Run global_cycle consistency test on Hera. +# Run global_cycle consistency test on Ursa. # # Set $WORK_DIR to your working directory. Set the project code # and queue as appropriate. @@ -27,7 +27,7 @@ module use ../../modulefiles module load build.$target.$compiler module list -WORK_DIR="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}" +WORK_DIR="${WORK_DIR:-/scratch4/NCEPDEV/stmp/$LOGNAME}" PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}" QUEUE="${QUEUE:-batch}" @@ -45,7 +45,7 @@ fi DATA_DIR="${WORK_DIR}/reg-tests/global-cycle" -export HOMEreg=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/global_cycle +export HOMEreg=/scratch3/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/global_cycle export OMP_NUM_THREADS_CY=2 From 1dfb9d07eb339e4662f55c8e6a502ef6af3e6fa5 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Mon, 9 Jun 2025 20:20:19 +0000 Subject: [PATCH 15/54] Update weight_gen utility script. Fixes #1064. --- util/weight_gen/{run.hera.sh => run.ursa.sh} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename util/weight_gen/{run.hera.sh => run.ursa.sh} (91%) diff --git a/util/weight_gen/run.hera.sh b/util/weight_gen/run.ursa.sh similarity index 91% rename from util/weight_gen/run.hera.sh rename to util/weight_gen/run.ursa.sh index 5e216ba80..052e92418 100755 --- a/util/weight_gen/run.hera.sh +++ b/util/weight_gen/run.ursa.sh @@ -11,7 +11,7 @@ #------------------------------------------------------------------------------- # -# Run the weight_gen program on Hera. +# Run the weight_gen program on Ursa. # # Set WORK_DIR to your working directory. # @@ -39,7 +39,7 @@ module list export CRES="C48" -export WORK_DIR=/scratch2/NCEPDEV/stmp1/$USER/weight_gen +export WORK_DIR=/scratch4/NCEPDEV/stmp/$USER/weight_gen ${UFS_DIR}/util/weight_gen/weight_gen.sh From bf0e27ca1e96a510ee9313222ec2af2d7d54b36d Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Mon, 9 Jun 2025 20:23:14 +0000 Subject: [PATCH 16/54] Remove hera from ./sorc/machine-setup.sh. Fixes #1064. --- sorc/machine-setup.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index 52dba04bc..3d712ca4d 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -38,14 +38,6 @@ elif [[ -d /opt/spack-stack ]] ; then fi target=container module purge -elif [[ -d /scratch1 ]] ; then - # We are on NOAA Hera - if ( ! eval module help > /dev/null 2>&1 ) ; then - echo load the module command 1>&2 - source /apps/lmod/lmod/init/$__ms_shell - fi - target=hera - module purge elif [[ -d /scratch3 ]] ; then # We are on NOAA Ursa if ( ! eval module help > /dev/null 2>&1 ) ; then From 2301081a105ef1fa3195106acabbef26181bf2dd Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Wed, 11 Jun 2025 19:21:38 +0000 Subject: [PATCH 17/54] Update the datmmesh_gen and sfc_climo_gen utilities. Fixes #1064. --- util/datmmesh_gen/{run.hera.sh => run.ursa.sh} | 6 +++--- util/sfc_climo_gen/{run.hera.sh => run.ursa.sh} | 6 +++--- util/sfc_climo_gen/sfc_gen.sh | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) rename util/datmmesh_gen/{run.hera.sh => run.ursa.sh} (89%) rename util/sfc_climo_gen/{run.hera.sh => run.ursa.sh} (85%) diff --git a/util/datmmesh_gen/run.hera.sh b/util/datmmesh_gen/run.ursa.sh similarity index 89% rename from util/datmmesh_gen/run.hera.sh rename to util/datmmesh_gen/run.ursa.sh index 770b90fb4..a54c42615 100755 --- a/util/datmmesh_gen/run.hera.sh +++ b/util/datmmesh_gen/run.ursa.sh @@ -3,7 +3,7 @@ # Edit account (-A) setting as required ! #SBATCH -J datmmesh_gen -#SBATCH -A nems +#SBATCH -A fv3-cpu #SBATCH --open-mode=truncate #SBATCH -o log #SBATCH -e log @@ -13,7 +13,7 @@ #------------------------------------------------------------------------------- # -# Run the datmmesh generation program on Hera. +# Run the datmmesh generation program on Ursa. # # Set NX and NY to your desired resolution. Valid choices are: # NX=3072, NY=1536 => 3072x1536 mesh @@ -34,7 +34,7 @@ module use $UFS_DIR/modulefiles module load build.$target.intelllvm module list -export OUTPUT_DIR=/scratch1/NCEPDEV/stmp4/$USER/datmmesh_gen +export OUTPUT_DIR=/scratch4/NCEPDEV/stmp/$USER/datmmesh_gen mkdir -p $OUTPUT_DIR export NX=3072 diff --git a/util/sfc_climo_gen/run.hera.sh b/util/sfc_climo_gen/run.ursa.sh similarity index 85% rename from util/sfc_climo_gen/run.hera.sh rename to util/sfc_climo_gen/run.ursa.sh index 697d0f350..91df6baa2 100755 --- a/util/sfc_climo_gen/run.hera.sh +++ b/util/sfc_climo_gen/run.ursa.sh @@ -1,7 +1,7 @@ #!/bin/bash #------------------------------------------------------------ -# Run the sfc_climo_gen program stand-alone on Hera using +# Run the sfc_climo_gen program stand-alone on Ursa using # pre-exiting 'grid' and 'orography' files. See the # sfc_gen.sh script for details. # @@ -15,9 +15,9 @@ #SBATCH -o log #SBATCH -e log #SBATCH --nodes=1 --ntasks-per-node=24 -#SBATCH --partition=bigmem +#SBATCH --mem=300g #SBATCH -q debug -#SBATCH -t 00:10:00 +#SBATCH -t 00:15:00 set -x diff --git a/util/sfc_climo_gen/sfc_gen.sh b/util/sfc_climo_gen/sfc_gen.sh index 762fd11f4..304e733e1 100755 --- a/util/sfc_climo_gen/sfc_gen.sh +++ b/util/sfc_climo_gen/sfc_gen.sh @@ -81,12 +81,12 @@ export veg_type_src="viirs.v3.igbp.30s" export soil_type_src="bnu.v3.30s" -export WORK_DIR=/scratch1/NCEPDEV/stmp2/$LOGNAME/work.sfc -export SAVE_DIR=/scratch1/NCEPDEV/stmp2/$LOGNAME/sfc.save +export WORK_DIR=/scratch4/NCEPDEV/stmp/$LOGNAME/work.sfc +export SAVE_DIR=/scratch4/NCEPDEV/stmp/$LOGNAME/sfc.save export FIX_FV3=${BASE_DIR}/fix/orog/C${res} -# Requires much more resources when true. On hera, use 6 nodes, +# Requires much more resources when true. On ursa, use 6 nodes, # 12 tasks per node. On WCOSS2, use 5 nodes, 12 tasks per node. export vegsoilt_frac=.false. From 9c3aa54bccbcdcebd218ef37d0dfde811f022536 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Wed, 11 Jun 2025 20:29:33 +0000 Subject: [PATCH 18/54] Begin updates to gdas_init utility. Fixes #1064. --- .../{driver.hera.sh => driver.usra.sh} | 55 +++++++++---------- util/gdas_init/set_fixed_files.sh | 4 +- 2 files changed, 28 insertions(+), 31 deletions(-) rename util/gdas_init/{driver.hera.sh => driver.usra.sh} (67%) diff --git a/util/gdas_init/driver.hera.sh b/util/gdas_init/driver.usra.sh similarity index 67% rename from util/gdas_init/driver.hera.sh rename to util/gdas_init/driver.usra.sh index faa244d07..e29f1b205 100755 --- a/util/gdas_init/driver.hera.sh +++ b/util/gdas_init/driver.usra.sh @@ -1,7 +1,7 @@ #!/bin/bash #--------------------------------------------------------------------- -# Driver script for running on Hera. +# Driver script for running on Ursa. # # Edit the 'config' file before running. #--------------------------------------------------------------------- @@ -12,16 +12,13 @@ compiler=${compiler:-"intelllvm"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.$compiler +module load prod_util/2.1.1 module list -# Needed for NDATE utility -module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles -module load prod_util/1.1.0 - PROJECT_CODE=fv3-cpu QUEUE=batch -export machine=hera +export machine=ursa source config @@ -35,75 +32,75 @@ if [ $EXTRACT_DATA == yes ]; then case $gfs_ver in v12 | v13) - DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ + DATAH=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_pre-v14.data.sh ${CDUMP}) DEPEND="-d afterok:$DATAH" if [ "$CDUMP" = "gdas" ] ; then - DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_enkf \ + DATA1=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_enkf \ -o log.data.enkf -e log.data.enkf ./get_pre-v14.data.sh enkf) DEPEND="-d afterok:$DATAH:$DATA1" fi ;; v14) - DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ + DATAH=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v14.data.sh ${CDUMP}) DEPEND="-d afterok:$DATAH" if [ "$CDUMP" = "gdas" ] ; then - DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_enkf \ + DATA1=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_enkf \ -o log.data.enkf -e log.data.enkf ./get_v14.data.sh enkf) DEPEND="-d afterok:$DATAH:$DATA1" fi ;; v15) - DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ + DATAH=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v15.data.sh ${CDUMP}) if [ "$CDUMP" = "gfs" ] ; then DEPEND="-d afterok:$DATAH" else - DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp1 \ + DATA1=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp1 \ -o log.data.grp1 -e log.data.grp1 ./get_v15.data.sh grp1) - DATA2=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp2 \ + DATA2=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp2 \ -o log.data.grp2 -e log.data.grp2 ./get_v15.data.sh grp2) - DATA3=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp3 \ + DATA3=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp3 \ -o log.data.grp3 -e log.data.grp3 ./get_v15.data.sh grp3) - DATA4=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp4 \ + DATA4=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp4 \ -o log.data.grp4 -e log.data.grp4 ./get_v15.data.sh grp4) - DATA5=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp5 \ + DATA5=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp5 \ -o log.data.grp5 -e log.data.grp5 ./get_v15.data.sh grp5) - DATA6=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp6 \ + DATA6=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp6 \ -o log.data.grp6 -e log.data.grp6 ./get_v15.data.sh grp6) - DATA7=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp7 \ + DATA7=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp7 \ -o log.data.grp7 -e log.data.grp7 ./get_v15.data.sh grp7) - DATA8=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp8 \ + DATA8=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp8 \ -o log.data.grp8 -e log.data.grp8 ./get_v15.data.sh grp8) DEPEND="-d afterok:$DATAH:$DATA1:$DATA2:$DATA3:$DATA4:$DATA5:$DATA6:$DATA7:$DATA8" fi ;; v16retro) - DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_v16retro \ + DATAH=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_v16retro \ -o log.data.v16retro -e log.data.v16retro ./get_v16retro.data.sh ${CDUMP}) DEPEND="-d afterok:$DATAH" ;; v16) - DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ + DATAH=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v16.data.sh ${CDUMP}) DEPEND="-d afterok:$DATAH" if [ "$CDUMP" = "gdas" ] ; then - DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp1 \ + DATA1=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp1 \ -o log.data.grp1 -e log.data.grp1 ./get_v16.data.sh grp1) - DATA2=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp2 \ + DATA2=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp2 \ -o log.data.grp2 -e log.data.grp2 ./get_v16.data.sh grp2) - DATA3=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp3 \ + DATA3=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp3 \ -o log.data.grp3 -e log.data.grp3 ./get_v16.data.sh grp3) - DATA4=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp4 \ + DATA4=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp4 \ -o log.data.grp4 -e log.data.grp4 ./get_v16.data.sh grp4) - DATA5=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp5 \ + DATA5=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp5 \ -o log.data.grp5 -e log.data.grp5 ./get_v16.data.sh grp5) - DATA6=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp6 \ + DATA6=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp6 \ -o log.data.grp6 -e log.data.grp6 ./get_v16.data.sh grp6) - DATA7=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp7 \ + DATA7=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp7 \ -o log.data.grp7 -e log.data.grp7 ./get_v16.data.sh grp7) - DATA8=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp8 \ + DATA8=$(sbatch --parsable --partition=u1-service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp8 \ -o log.data.grp8 -e log.data.grp8 ./get_v16.data.sh grp8) DEPEND="-d afterok:$DATAH:$DATA1:$DATA2:$DATA3:$DATA4:$DATA5:$DATA6:$DATA7:$DATA8" fi diff --git a/util/gdas_init/set_fixed_files.sh b/util/gdas_init/set_fixed_files.sh index d66098599..4ed90db2d 100755 --- a/util/gdas_init/set_fixed_files.sh +++ b/util/gdas_init/set_fixed_files.sh @@ -37,8 +37,8 @@ ORO_NAME="${CTAR}.mx${OCNRES}_oro_data" # latest orog version (20231027), need to use a previous version. #--------------------------------------------------------------------------- -if [ "$machine" = 'hera' ] ; then - FIX_ORO_INPUT=/scratch1/NCEPDEV/global/glopara/fix/orog/20230615 +if [ "$machine" = 'ursa' ] ; then + FIX_ORO_INPUT=/scratch3/NCEPDEV/global/role.glopara/fix/orog/20230615 elif [ "$machine" = 'wcoss2' ] ; then FIX_ORO_INPUT=/lfs/h2/emc/global/noscrub/emc.global/FIX/fix/orog/20230615 elif [ "$machine" = 'jet' ] ; then From a905887596d2d804069c807b8c6eafae9bfa6da8 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 12 Jun 2025 18:26:20 +0000 Subject: [PATCH 19/54] Finish modifications to gdas_init. Fixes #1064. --- cmake/{mpiexec.hera => mpiexec.ursa} | 0 .../{driver.usra.sh => driver.ursa.sh} | 29 ++++++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) rename cmake/{mpiexec.hera => mpiexec.ursa} (100%) rename util/gdas_init/{driver.usra.sh => driver.ursa.sh} (88%) diff --git a/cmake/mpiexec.hera b/cmake/mpiexec.ursa similarity index 100% rename from cmake/mpiexec.hera rename to cmake/mpiexec.ursa diff --git a/util/gdas_init/driver.usra.sh b/util/gdas_init/driver.ursa.sh similarity index 88% rename from util/gdas_init/driver.usra.sh rename to util/gdas_init/driver.ursa.sh index e29f1b205..ceef06af3 100755 --- a/util/gdas_init/driver.usra.sh +++ b/util/gdas_init/driver.ursa.sh @@ -116,13 +116,14 @@ fi # extract data? if [ $RUN_CHGRES == yes ]; then export APRUN=srun - NODES=3 + NODES=2 WALLT="0:15:00" + MEM=300G export OMP_NUM_THREADS=1 if [ $CRES_HIRES == 'C768' ] ; then - NODES=5 + NODES=3 elif [ $CRES_HIRES == 'C1152' ] ; then - NODES=8 + NODES=4 WALLT="0:20:00" fi case $gfs_ver in @@ -130,28 +131,28 @@ if [ $RUN_CHGRES == yes ]; then export OMP_NUM_THREADS=4 export OMP_STACKSIZE=1024M sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} --cpus-per-task=$OMP_NUM_THREADS \ - -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ + --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_pre-v14.chgres.sh ${CDUMP} ;; v14) - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ + sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v14.chgres.sh ${CDUMP} ;; v15) - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ + sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v15.chgres.sh ${CDUMP} ;; v16retro) if [ "$CDUMP" = "gdas" ] ; then - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ + sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16retro.chgres.sh hires else - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ + sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16.chgres.sh ${CDUMP} fi ;; v16) - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ + sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16.chgres.sh ${CDUMP} ;; esac @@ -162,7 +163,7 @@ if [ $RUN_CHGRES == yes ]; then if [ "$gfs_ver" = "v16retro" ]; then - sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_enkf \ + sbatch --parsable --ntasks-per-node=12 --nodes=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_enkf \ -o log.enkf -e log.enkf ${DEPEND} run_v16retro.chgres.sh enkf else @@ -179,19 +180,19 @@ if [ $RUN_CHGRES == yes ]; then export OMP_NUM_THREADS=2 export OMP_STACKSIZE=1024M sbatch --parsable --ntasks-per-node=12 --nodes=1 --cpus-per-task=$OMP_NUM_THREADS \ - -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ + --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_pre-v14.chgres.sh ${MEMBER_CH} ;; v14) - sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ + sbatch --parsable --ntasks-per-node=12 --nodes=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v14.chgres.sh ${MEMBER_CH} ;; v15) - sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ + sbatch --parsable --ntasks-per-node=12 --nodes=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v15.chgres.sh ${MEMBER_CH} ;; v16) - sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ + sbatch --parsable --ntasks-per-node=12 --nodes=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v16.chgres.sh ${MEMBER_CH} ;; esac From d94b1c82e6cd7b9ebdb869b9030c2786f30282d5 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 12 Jun 2025 18:33:23 +0000 Subject: [PATCH 20/54] Update README.md. Fixes #1064. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3901548a2..6495c6fda 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ It also uses the following repositories: ## Installing -On Orion, Hercules, Jet, Hera, S4, Gaea and WCOSS2 do the following: +On Orion, Hercules, Jet, Ursa, S4, Gaea and WCOSS2 do the following: 1) Set the 'fixed' directories using the `link_fixdirs.sh` script in `./fix`. Usage: `./link_fixdirs.sh $RUN_ENVIR $machine`, @@ -68,7 +68,7 @@ where `$RUN_ENVIR` is "emc" or "nco" (most developers should choose "emc") and `$machine` is the platform. Example: ``` -./link_fixdirs.sh emc hera +./link_fixdirs.sh emc ursa ``` 2) Then, invoke the build script: From 246b19cd22c2010ed3195feb13a558555e778e6b Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 12 Jun 2025 18:38:20 +0000 Subject: [PATCH 21/54] Update local unit tests. Fixes #1064. --- cmake/LibMPI.cmake | 22 +++++++--------------- cmake/mpiexec.ursa | 2 +- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/cmake/LibMPI.cmake b/cmake/LibMPI.cmake index 83dbe8df8..02278ef00 100644 --- a/cmake/LibMPI.cmake +++ b/cmake/LibMPI.cmake @@ -29,21 +29,13 @@ function (platform_name RETURN_VARIABLE) # Determine platform name from site name... site_name (SITENAME) - # hera - if (SITENAME MATCHES "^hfe01" OR - SITENAME MATCHES "^hfe02" OR - SITENAME MATCHES "^hfe03" OR - SITENAME MATCHES "^hfe04" OR - SITENAME MATCHES "^hfe05" OR - SITENAME MATCHES "^hfe06" OR - SITENAME MATCHES "^hfe07" OR - SITENAME MATCHES "^hfe08" OR - SITENAME MATCHES "^hfe09" OR - SITENAME MATCHES "^hfe10" OR - SITENAME MATCHES "^hfe11" OR - SITENAME MATCHES "^hfe12") - - set (${RETURN_VARIABLE} "hera" PARENT_SCOPE) + # ursa + if (SITENAME MATCHES "^ufe01" OR + SITENAME MATCHES "^ufe02" OR + SITENAME MATCHES "^ufe03" OR + SITENAME MATCHES "^ufe04") + + set (${RETURN_VARIABLE} "ursa" PARENT_SCOPE) # wcoss2 elseif (SITENAME MATCHES "^along01" OR diff --git a/cmake/mpiexec.ursa b/cmake/mpiexec.ursa index 23bec1047..798aaf04d 100755 --- a/cmake/mpiexec.ursa +++ b/cmake/mpiexec.ursa @@ -12,4 +12,4 @@ QOS=debug NP=$1 shift -srun -A $ACCOUNT -q $QOS -n $NP $@ +srun -A $ACCOUNT -q $QOS -t 0:01 -n $NP $@ From 9054d18e3aac4bbe756e4a3f7b738ca75777a804 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 12 Jun 2025 20:09:49 +0000 Subject: [PATCH 22/54] Update grid driver script. Fixes #1064. --- .../{driver_grid.hera.sh => driver_grid.ursa.sh} | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) rename driver_scripts/{driver_grid.hera.sh => driver_grid.ursa.sh} (95%) diff --git a/driver_scripts/driver_grid.hera.sh b/driver_scripts/driver_grid.ursa.sh similarity index 95% rename from driver_scripts/driver_grid.hera.sh rename to driver_scripts/driver_grid.ursa.sh index e990ac948..0e73007b0 100755 --- a/driver_scripts/driver_grid.hera.sh +++ b/driver_scripts/driver_grid.ursa.sh @@ -5,13 +5,13 @@ #SBATCH --open-mode=truncate #SBATCH -o log.fv3_grid_driver #SBATCH -e log.fv3_grid_driver -#SBATCH --nodes=6 --ntasks-per-node=12 -#SBATCH --partition=bigmem +#SBATCH --nodes=4 --ntasks-per-node=12 +#SBATCH --mem=300g #SBATCH -q debug #SBATCH -t 00:30:00 #----------------------------------------------------------------------- -# Driver script to create a cubic-sphere based model grid on Hera. +# Driver script to create a cubic-sphere based model grid on Ursa. # # Produces the following files (netcdf, each tile in separate file): # 1) 'mosaic' and 'grid' files containing lat/lon and other @@ -24,9 +24,8 @@ # Note: The sfc_climo_gen program only runs with an # mpi task count that is a multiple of six. This is # an ESMF library requirement. Large grids may require -# tasks spread across multiple nodes or to be run on -# 'bigmem' nodes (#SBATCH --partition=bigmem). The -# orography code benefits from threads. +# tasks spread across multiple nodes. The orography +# code benefits from threads. # # To run, do the following: # @@ -158,8 +157,8 @@ fi #----------------------------------------------------------------------- export home_dir=$SLURM_SUBMIT_DIR/.. -export TEMP_DIR=/scratch2/NCEPDEV/stmp1/$LOGNAME/fv3_grid.$gtype -export out_dir=/scratch2/NCEPDEV/stmp1/$LOGNAME/my_grids +export TEMP_DIR=/scratch4/NCEPDEV/stmp/$LOGNAME/fv3_grid.$gtype +export out_dir=/scratch4/NCEPDEV/stmp/$LOGNAME/my_grids #----------------------------------------------------------------------- # Should not need to change anything below here. From 867b42ca75474804bf56cc2573353fc948dcf140 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 12 Jun 2025 20:13:31 +0000 Subject: [PATCH 23/54] Remove hera build modules. Add ursa build module using gnu. Fixes #1064. --- modulefiles/build.hera.gnu.lua | 64 ------------------------ modulefiles/build.hera.intel.lua | 65 ------------------------ modulefiles/build.hera.intelllvm.lua | 74 ---------------------------- modulefiles/build.ursa.gnu.lua | 60 ++++++++++++++++++++++ 4 files changed, 60 insertions(+), 203 deletions(-) delete mode 100644 modulefiles/build.hera.gnu.lua delete mode 100644 modulefiles/build.hera.intel.lua delete mode 100644 modulefiles/build.hera.intelllvm.lua create mode 100644 modulefiles/build.ursa.gnu.lua diff --git a/modulefiles/build.hera.gnu.lua b/modulefiles/build.hera.gnu.lua deleted file mode 100644 index 54dcb55de..000000000 --- a/modulefiles/build.hera.gnu.lua +++ /dev/null @@ -1,64 +0,0 @@ -help([[ -Load environment to compile UFS_UTILS on Hera using Gnu 13.3 -]]) - -hpss_ver=os.getenv("hpss_ver") or "" -load(pathJoin("hpss", hpss_ver)) - -prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/installs/gnu/modulefiles") -prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/installs/openmpi/modulefiles") -prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/spack-stack/spack-stack-1.6.0_gnu13/envs/fms-2024.01/install/modulefiles/Core") - -stack_gnu_ver=os.getenv("stack_gnu_ver") or "13.3.0" -load(pathJoin("stack-gcc", stack_gnu_ver)) - -stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.6" -load(pathJoin("stack-openmpi", stack_openmpi_ver)) - -cmake_ver=os.getenv("cmake_ver") or "3.23.1" -load(pathJoin("cmake", cmake_ver)) - -bacio_ver=os.getenv("bacio_ver") or "2.4.1" -load(pathJoin("bacio", bacio_ver)) - -g2_ver=os.getenv("g2_ver") or "3.5.1" -load(pathJoin("g2", g2_ver)) - -ip_ver=os.getenv("ip_ver") or "4.3.0" -load(pathJoin("ip", ip_ver)) - -nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" -load(pathJoin("nemsio", nemsio_ver)) - -sp_ver=os.getenv("sp_ver") or "2.5.0" -load(pathJoin("sp", sp_ver)) - -w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" -load(pathJoin("w3emc", w3emc_ver)) - --- Uncomment when CHGRES_ALL is ON ---sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" ---load(pathJoin("sfcio", sfcio_ver)) - -sigio_ver=os.getenv("sigio_ver") or "2.3.2" -load(pathJoin("sigio", sigio_ver)) - -nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" -load(pathJoin("nccmp", nccmp_ver)) - -esmf_ver=os.getenv("esmf_ver") or "8.6.0" -load(pathJoin("esmf", esmf_ver)) - -nco_ver=os.getenv("nco_ver") or "5.0.6" -load(pathJoin("nco", nco_ver)) - - -prepend_path("CPPFLAGS", " -I/apps/slurm_hera/23.11.3/include/slurm"," ") -prepend_path("LD_LIBRARY_PATH", "/apps/slurm_hera/23.11.3/lib") -setenv("LD_PRELOAD", "/scratch2/NCEPDEV/stmp1/role.epic/installs/gnu/13.3.0/lib64/libstdc++.so.6") - -setenv("CC", "mpicc") -setenv("CXX", "mpic++") -setenv("FC", "mpif90") - -whatis("Description: UFS_UTILS build environment") diff --git a/modulefiles/build.hera.intel.lua b/modulefiles/build.hera.intel.lua deleted file mode 100644 index f79d67adb..000000000 --- a/modulefiles/build.hera.intel.lua +++ /dev/null @@ -1,65 +0,0 @@ -help([[ -Load environment to compile UFS_UTILS on Hera using Intel -]]) - -hpss_ver=os.getenv("hpss_ver") or "" -load(pathJoin("hpss", hpss_ver)) - -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") - -stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" -load(pathJoin("stack-intel", stack_intel_ver)) - -stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" -load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) - -cmake_ver=os.getenv("cmake_ver") or "3.23.1" -load(pathJoin("cmake", cmake_ver)) - -bacio_ver=os.getenv("bacio_ver") or "2.4.1" -load(pathJoin("bacio", bacio_ver)) - -g2_ver=os.getenv("g2_ver") or "3.5.1" -load(pathJoin("g2", g2_ver)) - -ip_ver=os.getenv("ip_ver") or "4.3.0" -load(pathJoin("ip", ip_ver)) - -nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" -load(pathJoin("nemsio", nemsio_ver)) - -sp_ver=os.getenv("sp_ver") or "2.5.0" -load(pathJoin("sp", sp_ver)) - -w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" -load(pathJoin("w3emc", w3emc_ver)) - --- Uncomment when CHGRES_ALL is ON ---sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" ---load(pathJoin("sfcio", sfcio_ver)) - -sigio_ver=os.getenv("sigio_ver") or "2.3.2" -load(pathJoin("sigio", sigio_ver)) - -zlib_ver=os.getenv("zlib_ver") or "1.2.13" -load(pathJoin("zlib", zlib_ver)) - -png_ver=os.getenv("png_ver") or "1.6.37" -load(pathJoin("libpng", png_ver)) - -netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" -load(pathJoin("netcdf-c", netcdf_c_ver)) - -netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1" -load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) - -nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" -load(pathJoin("nccmp", nccmp_ver)) - -esmf_ver=os.getenv("esmf_ver") or "8.6.0" -load(pathJoin("esmf", esmf_ver)) - -nco_ver=os.getenv("nco_ver") or "5.0.6" -load(pathJoin("nco", nco_ver)) - -whatis("Description: UFS_UTILS build environment") diff --git a/modulefiles/build.hera.intelllvm.lua b/modulefiles/build.hera.intelllvm.lua deleted file mode 100644 index dc59c40f5..000000000 --- a/modulefiles/build.hera.intelllvm.lua +++ /dev/null @@ -1,74 +0,0 @@ -help([[ -Load environment to compile UFS_UTILS on Hera using Intel -]]) - -hpss_ver=os.getenv("hpss_ver") or "" -load(pathJoin("hpss", hpss_ver)) - -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") - -stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" -load(pathJoin("stack-intel", stack_intel_ver)) - -stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" -load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) - -load("gnu") -load("intel/2024.2.1") - -cmake_ver=os.getenv("cmake_ver") or "3.23.1" -load(pathJoin("cmake", cmake_ver)) - -bacio_ver=os.getenv("bacio_ver") or "2.4.1" -load(pathJoin("bacio", bacio_ver)) - -g2_ver=os.getenv("g2_ver") or "3.5.1" -load(pathJoin("g2", g2_ver)) - -ip_ver=os.getenv("ip_ver") or "4.3.0" -load(pathJoin("ip", ip_ver)) - -nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" -load(pathJoin("nemsio", nemsio_ver)) - -sp_ver=os.getenv("sp_ver") or "2.5.0" -load(pathJoin("sp", sp_ver)) - -w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" -load(pathJoin("w3emc", w3emc_ver)) - --- Uncomment when CHGRES_ALL is ON ---sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" ---load(pathJoin("sfcio", sfcio_ver)) - -sigio_ver=os.getenv("sigio_ver") or "2.3.2" -load(pathJoin("sigio", sigio_ver)) - -zlib_ver=os.getenv("zlib_ver") or "1.2.13" -load(pathJoin("zlib", zlib_ver)) - -png_ver=os.getenv("png_ver") or "1.6.37" -load(pathJoin("libpng", png_ver)) - -netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" -load(pathJoin("netcdf-c", netcdf_c_ver)) - -netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1" -load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) - -nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" -load(pathJoin("nccmp", nccmp_ver)) - -esmf_ver=os.getenv("esmf_ver") or "8.6.0" -load(pathJoin("esmf", esmf_ver)) - -nco_ver=os.getenv("nco_ver") or "5.0.6" -load(pathJoin("nco", nco_ver)) - -setenv("I_MPI_CC", "icx") -setenv("I_MPI_FC", "ifx") - -setenv("CC", "mpicc") -setenv("FC", "mpifc") - -whatis("Description: UFS_UTILS build environment") diff --git a/modulefiles/build.ursa.gnu.lua b/modulefiles/build.ursa.gnu.lua new file mode 100644 index 000000000..05fd0ab20 --- /dev/null +++ b/modulefiles/build.ursa.gnu.lua @@ -0,0 +1,60 @@ +help([[ +Load environment to compile UFS_UTILS on Ursa using Gnu +]]) + +prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.9.1/envs/ue-gcc-12.4.0/install/modulefiles/Core") + +stack_gcc_ver=os.getenv("stack_gcc_ver") or "12.4.0" +load(pathJoin("stack-gcc", stack_gcc_ver)) + +stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.6" +load(pathJoin("stack-openmpi", stack_openmpi_ver)) + +cmake_ver=os.getenv("cmake_ver") or "3.30.2" +load(pathJoin("cmake", cmake_ver)) + +bacio_ver=os.getenv("bacio_ver") or "2.4.1" +load(pathJoin("bacio", bacio_ver)) + +g2_ver=os.getenv("g2_ver") or "3.5.1" +load(pathJoin("g2", g2_ver)) + +ip_ver=os.getenv("ip_ver") or "5.1.0" +load(pathJoin("ip", ip_ver)) + +nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" +load(pathJoin("nemsio", nemsio_ver)) + +--sp_ver=os.getenv("sp_ver") or "2.5.0" +--load(pathJoin("sp", sp_ver)) + +w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" +load(pathJoin("w3emc", w3emc_ver)) + +-- Uncomment when CHGRES_ALL is ON +--sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" +--load(pathJoin("sfcio", sfcio_ver)) + +sigio_ver=os.getenv("sigio_ver") or "2.3.3" +load(pathJoin("sigio", sigio_ver)) + +netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" +load(pathJoin("netcdf-c", netcdf_c_ver)) + +netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1" +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) + +nccmp_ver=os.getenv("nccmp_ver") or "1.9.1.0" +load(pathJoin("nccmp", nccmp_ver)) + +esmf_ver=os.getenv("esmf_ver") or "8.6.1" +load(pathJoin("esmf", esmf_ver)) + +nco_ver=os.getenv("nco_ver") or "5.2.4" +load(pathJoin("nco", nco_ver)) + +setenv("MPI_CC", "mpicc") +setenv("MPI_CCX", "mpic++") +setenv("MPI_FC", "mpif90") + +whatis("Description: UFS_UTILS build environment") From ad51d6d565591e5d65459119d0ff9b89abffe5ab Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 13 Jun 2025 12:25:55 +0000 Subject: [PATCH 24/54] Update comments in CMakeLists.txt and build_all.sh. Fixes #1064. --- CMakeLists.txt | 2 -- build_all.sh | 1 - 2 files changed, 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index daf8948e7..2a27d2ccb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,8 +72,6 @@ if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel|IntelLLVM)$") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -traceback") set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -fp-model precise") set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check all -ftrapuv") -# On some machines (Hera), using -check all results in an error -# at the linking step. The uninit check is the culprit. if(CMAKE_Fortran_COMPILER_ID MATCHES "^(IntelLLVM)$") set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -check nouninit") endif() diff --git a/build_all.sh b/build_all.sh index 7012cd152..7ed86a7f8 100755 --- a/build_all.sh +++ b/build_all.sh @@ -58,7 +58,6 @@ CMAKE_FLAGS+=" -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX:-${DIR_ROOT}} -DCMAKE_INS # Testing options # The unit test data download is part of the build system. Not all machines can # access the EMC ftp site, so turn off the build (-DBUILD_TESTING=OFF) of the units tests accordingly. -# Those with access to the EMC ftp site are: Orion and Hera. CMAKE_FLAGS+=" -DBUILD_TESTING=${BUILD_TESTING:-OFF}" # Allow users of this script to provide CMake options e.g. -DGFS=ON|OFF to build GFS specific utilities only From 14a3401827b70294c2c2e3003de71e7655d851cb Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 13 Jun 2025 12:30:02 +0000 Subject: [PATCH 25/54] Convert the orog utilities to run on ursa. Fixes #1064. --- .../orog_netcdf_tools.fd/mask.fd/mask.f90 | 2 +- .../orog_netcdf_tools.fd/mask.fd/runit.sh | 7 ++++--- .../orog_netcdf_tools.fd/ramp.fd/ramp.f90 | 2 +- .../orog_netcdf_tools.fd/ramp.fd/runit.sh | 7 ++++--- .../orog_netcdf_tools.fd/topo.fd/runit.sh | 7 ++++--- .../orog_netcdf_tools.fd/topo.fd/topo.f90 | 2 +- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/mask.fd/mask.f90 b/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/mask.fd/mask.f90 index e83bc6932..4557a3984 100644 --- a/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/mask.fd/mask.f90 +++ b/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/mask.fd/mask.f90 @@ -56,7 +56,7 @@ program mask_netcdf print*,'lat_corner ',i,lats_corner(i) enddo - fileraw="/scratch1/NCEPDEV/global/glopara/fix/raw/orog/landcover30.fixed" + fileraw="/scratch3/NCEPDEV/global/role.glopara/fix/raw/orog/landcover30.fixed" open(11, file=trim(fileraw), access='direct', recl=idim*jdim) read(11, rec=1, iostat=istat) mask diff --git a/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/mask.fd/runit.sh b/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/mask.fd/runit.sh index 8cc802868..82e21e451 100755 --- a/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/mask.fd/runit.sh +++ b/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/mask.fd/runit.sh @@ -1,8 +1,9 @@ #!/bin/sh -# Run on Hera. +# Run on Ursa. -#SBATCH --ntasks=1 --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --mem=25g #SBATCH -t 0:03:00 #SBATCH -A fv3-cpu #SBATCH -q debug @@ -14,7 +15,7 @@ set -x source ../../../machine-setup.sh > /dev/null 2>&1 module use ../../../../modulefiles -module load build.$target.intel +module load build.$target.intelllvm module list ../../../../exec/mask.exe diff --git a/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/ramp.fd/ramp.f90 b/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/ramp.fd/ramp.f90 index b897ddd16..8aa30c453 100644 --- a/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/ramp.fd/ramp.f90 +++ b/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/ramp.fd/ramp.f90 @@ -58,7 +58,7 @@ program ramp_netcdf print*,'lat_corner ',i,lats_corner(i) enddo - fileraw="/scratch1/NCEPDEV/global/glopara/fix/raw/orog/thirty.second.antarctic.new.bin" + fileraw="/scratch3/NCEPDEV/global/role.glopara/fix/raw/orog/thirty.second.antarctic.new.bin" open(11, file=trim(fileraw), form='unformatted', access='sequential', iostat=istat) print*,'iostat on open ',istat diff --git a/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/ramp.fd/runit.sh b/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/ramp.fd/runit.sh index f4739a1ff..e7fc46723 100755 --- a/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/ramp.fd/runit.sh +++ b/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/ramp.fd/runit.sh @@ -1,8 +1,9 @@ #!/bin/sh -# Run on Hera. +# Run on Ursa. -#SBATCH --ntasks=1 --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --mem=25g #SBATCH -t 0:03:00 #SBATCH -A fv3-cpu #SBATCH -q debug @@ -14,7 +15,7 @@ set -x source ../../../machine-setup.sh > /dev/null 2>&1 module use ../../../../modulefiles -module load build.$target.intel +module load build.$target.intelllvm module list ../../../../exec/ramp.exe diff --git a/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/topo.fd/runit.sh b/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/topo.fd/runit.sh index 5e038c83d..869143076 100755 --- a/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/topo.fd/runit.sh +++ b/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/topo.fd/runit.sh @@ -1,8 +1,9 @@ #!/bin/sh -# Run on Hera. +# Run on Ursa. -#SBATCH --ntasks=1 --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --mem=50g #SBATCH -t 0:03:00 #SBATCH -A fv3-cpu #SBATCH -q debug @@ -14,7 +15,7 @@ set -x source ../../../machine-setup.sh > /dev/null 2>&1 module use ../../../../modulefiles -module load build.$target.intel +module load build.$target.intelllvm module list ../../../../exec/topo.exe diff --git a/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/topo.fd/topo.f90 b/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/topo.fd/topo.f90 index 2674ae063..a5b78735f 100644 --- a/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/topo.fd/topo.f90 +++ b/sorc/orog_mask_tools.fd/orog_netcdf_tools.fd/topo.fd/topo.f90 @@ -56,7 +56,7 @@ program topo_netcdf print*,'lat_corner ',i,lats_corner(i) enddo - fileraw="/scratch1/NCEPDEV/global/glopara/fix/raw/orog/gmted2010.30sec.int" + fileraw="/scratch3/NCEPDEV/global/role.glopara/fix/raw/orog/gmted2010.30sec.int" open(11, file=trim(fileraw), access='direct', recl=idim*jdim*2) read(11, rec=1, iostat=istat) topo From c33463ee4fa5b92e7cb0c67ccf10cdad45f72e25 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 13 Jun 2025 12:49:24 +0000 Subject: [PATCH 26/54] Update 'readthedocs'. Fixes #1064. --- docs/source/ufs_utils.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/ufs_utils.rst b/docs/source/ufs_utils.rst index bf29aeaef..2939e1d37 100644 --- a/docs/source/ufs_utils.rst +++ b/docs/source/ufs_utils.rst @@ -678,7 +678,7 @@ Introduction The gdas_init utility is used to create coldstart initial conditions for global cycled and forecast-only experiments using the chgres_cube program. It has two components: one that pulls the input data required by chgres_cube from HPSS, and one that runs chgres_cube. The utility is only supported on machines with access to HPSS: - * Hera + * Ursa * Jet * WCOSS2 * Gaea C6 @@ -698,7 +698,7 @@ Invoke the build script from the root directory: ./build_all.sh -Set the 'fixed' directories using the script in the './fix' subdirectory (where $MACHINE is 'hera', 'jet', 'orion', 'hercules', 'gaea', 'wcoss2', or 's4'): +Set the 'fixed' directories using the script in the './fix' subdirectory (where $MACHINE is "wcoss2", "ursa", "jet", "orion", "hercules", "s4", "gaeac5" or "gaeac6"): :: @@ -727,7 +727,7 @@ Note: This utility selects the ocean resolution in the set_fixed_files.sh script Kick off the utility -------------------- -Submit the driver script (where $MACHINE is 'hera', 'jet', 'wcoss2', or 's4') +Submit the driver script (where $MACHINE is 'gaeac6', 'ursa', 'jet', 'wcoss2', or 's4') :: From 5f5d605987aeb4c60a8a77abd27cf30b3c5a0295 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 13 Jun 2025 12:55:04 +0000 Subject: [PATCH 27/54] Update README.md. Fixes #1064. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6495c6fda..8e337e547 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ It also uses the following repositories: ## Installing -On Orion, Hercules, Jet, Ursa, S4, Gaea and WCOSS2 do the following: +On Orion, Hercules, Jet, Ursa, S4, Gaea c5, Gaea c6, and WCOSS2 do the following: 1) Set the 'fixed' directories using the `link_fixdirs.sh` script in `./fix`. Usage: `./link_fixdirs.sh $RUN_ENVIR $machine`, From 49998b4404d33c36328144629e399f9382bdc4b1 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 13 Jun 2025 13:44:07 +0000 Subject: [PATCH 28/54] Some final updates. Fixes #1064. --- .github/PULL_REQUEST_TEMPLATE | 1 - reg_tests/rt.sh | 10 +++++----- ush/fv3gfs_ocean_merge.sh | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index a99b830df..cef671b4b 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -46,4 +46,3 @@ If this PR is resolving or referencing one or more issues, in this repository or ## CONTRIBUTORS (optional): If others have contributed to this work aside from the PR author, list them here - diff --git a/reg_tests/rt.sh b/reg_tests/rt.sh index 4973921c4..12b7ce865 100755 --- a/reg_tests/rt.sh +++ b/reg_tests/rt.sh @@ -43,8 +43,8 @@ else target=Jet elif [[ -d /lfs/h1 ]] ; then target=WCOSS2 - elif [[ -d /scratch1 ]] ; then - target=Hera + elif [[ -d /scratch3 ]] ; then + target=Usra fi echo "Clone Failed" | mail -s "UFS_UTILS Consistency Tests failed on ${target}" ${MAILTO} fi @@ -88,8 +88,8 @@ cd ../reg_tests set -x -#if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hera" ]] || [[ $target == "hercules" ]] || [[ $target == "wcoss2" ]] ; then -#if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hera" ]] || [[ $target == "hercules" ]] ; then +#if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hercules" ]] || [[ $target == "wcoss2" ]] ; then +#if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hercules" ]] ; then cd regrid_sfc ./driver.sh @@ -127,7 +127,7 @@ done for dir in weight_gen ice_blend; do cd $dir - if [[ $target == "hera" ]] || [[ $target == "jet" ]] || [[ $target == "orion" ]] || [[ $target == "s4" ]] || [[ $target == "hercules" ]] ; then + if [[ $target == "ursa" ]] || [[ $target == "jet" ]] || [[ $target == "orion" ]] || [[ $target == "s4" ]] || [[ $target == "hercules" ]] ; then sbatch -A ${PROJECT_CODE} ./driver.$target.sh elif [[ $target == "wcoss2" ]] ; then qsub -v WORK_DIR ./driver.$target.sh diff --git a/ush/fv3gfs_ocean_merge.sh b/ush/fv3gfs_ocean_merge.sh index 519a2e9da..dd9583766 100755 --- a/ush/fv3gfs_ocean_merge.sh +++ b/ush/fv3gfs_ocean_merge.sh @@ -4,12 +4,12 @@ # # Check-out ufs-utils (and be sure to check out the ccpp submodule) # cd fix -# ./link_fixdirs.sh emc hera +# ./link_fixdirs.sh emc $MACHINE # cd ../reg_tests/cpld_gridgen # Edit the rt.conf and add the non-standard case(s) you want to generate # Note you need to keep/run the C384_025 case because the lower resolution cases depend on it to generate the remapping weights used to create the CICE ICs and/or do the Post jobs. # Edit rt.sh for proper accounts/partitions -# Build and run the test using ./rt.sh -b >output 2>&1 & +# Build and run the test using ./rt.sh -b # Your results should be in /scratch1/NCEPDEV/stmp4/First.Last/CPLD_GRIDGEN/rt_# # # From 1f71b1d02931586963866c8022091dd7f54420c9 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 13 Jun 2025 17:23:51 +0000 Subject: [PATCH 29/54] Remove 's4' support from gdas_init utility. Fixes #1064. --- util/gdas_init/driver.s4.sh | 127 ------------------------------ util/gdas_init/set_fixed_files.sh | 2 - 2 files changed, 129 deletions(-) delete mode 100755 util/gdas_init/driver.s4.sh diff --git a/util/gdas_init/driver.s4.sh b/util/gdas_init/driver.s4.sh deleted file mode 100755 index 5a1d7b9ab..000000000 --- a/util/gdas_init/driver.s4.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash - -#--------------------------------------------------------------------- -# Driver script for running on S4. -# -# Edit the 'config' file before running. -#--------------------------------------------------------------------- - -set -x - -compiler=${compiler:-"intel"} -source ../../sorc/machine-setup.sh > /dev/null 2>&1 -module use ../../modulefiles -module load build.$target.$compiler -module list - -# Needed for NDATE utility -module load prod_util/2.1.1 - -PROJECT_CODE=star -QUEUE=s4 - -source config - -export machine=s4 - -if [ $EXTRACT_DATA == yes ]; then - - echo "HPSS is not accessible from S4! If you wish to run just the chgres portion, set EXTRACT_DATA=NO in the config file and try again." - exit - -else # do not extract data. - - DEPEND=' ' - -fi # extract data? - -if [ $RUN_CHGRES == yes ]; then - - export APRUN=srun - NODES=3 - WALLT="0:15:00" - export OMP_NUM_THREADS=1 - if [ $CRES_HIRES == 'C768' ] ; then - NODES=5 - elif [ $CRES_HIRES == 'C1152' ] ; then - NODES=8 - WALLT="0:20:00" - fi - case $gfs_ver in - v12 | v13) - export OMP_NUM_THREADS=4 - export OMP_STACKSIZE=1024M - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} --cpus-per-task=$OMP_NUM_THREADS \ - -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ - -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_pre-v14.chgres.sh ${CDUMP} - ;; - v14) - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ - -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v14.chgres.sh ${CDUMP} - ;; - v15) - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ - -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v15.chgres.sh ${CDUMP} - ;; - v16retro) - if [ "$CDUMP" = "gdas" ] ; then - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ - -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16retro.chgres.sh hires - else - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ - -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16.chgres.sh ${CDUMP} - fi - ;; - v16) - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ - -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16.chgres.sh ${CDUMP} - ;; - esac - - if [ "$CDUMP" = "gdas" ]; then - - WALLT="0:15:00" - - if [ "$gfs_ver" = "v16retro" ]; then - - sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_enkf \ - -o log.enkf -e log.enkf ${DEPEND} run_v16retro.chgres.sh enkf - - else - - MEMBER=1 - while [ $MEMBER -le 80 ]; do - if [ $MEMBER -lt 10 ]; then - MEMBER_CH="00${MEMBER}" - else - MEMBER_CH="0${MEMBER}" - fi - case $gfs_ver in - v12 | v13) - export OMP_NUM_THREADS=2 - export OMP_STACKSIZE=1024M - sbatch --parsable --ntasks-per-node=12 --nodes=1 --cpus-per-task=$OMP_NUM_THREADS \ - -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ - -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_pre-v14.chgres.sh ${MEMBER_CH} - ;; - v14) - sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ - -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v14.chgres.sh ${MEMBER_CH} - ;; - v15) - sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ - -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v15.chgres.sh ${MEMBER_CH} - ;; - v16) - sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ - -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v16.chgres.sh ${MEMBER_CH} - ;; - esac - MEMBER=$(( $MEMBER + 1 )) - done - - fi # v16 retro? - - fi # which CDUMP? - -fi # run chgres? diff --git a/util/gdas_init/set_fixed_files.sh b/util/gdas_init/set_fixed_files.sh index 4ed90db2d..d8cc82397 100755 --- a/util/gdas_init/set_fixed_files.sh +++ b/util/gdas_init/set_fixed_files.sh @@ -43,8 +43,6 @@ elif [ "$machine" = 'wcoss2' ] ; then FIX_ORO_INPUT=/lfs/h2/emc/global/noscrub/emc.global/FIX/fix/orog/20230615 elif [ "$machine" = 'jet' ] ; then FIX_ORO_INPUT=/lfs5/HFIP/hfv3gfs/glopara/FIX/fix/orog/20230615 -elif [ "$machine" = 's4' ] ; then - FIX_ORO_INPUT=/data/prod/glopara/fix/orog/20230615 elif [ "$machine" = 'gaeac6' ] ; then FIX_ORO_INPUT=/gpfs/f6/drsa-precip3/world-shared/role.glopara/fix/orog/20230615 else From cf4b831ee9ac338c5991ea8801ef41a06ed90a81 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 13 Jun 2025 17:41:16 +0000 Subject: [PATCH 30/54] Remove more 's4' support. Fixes #1064. --- README.md | 2 +- cmake/LibMPI.cmake | 4 - docs/source/ufs_utils.rst | 7 +- fix/link_fixdirs.sh | 10 +- modulefiles/build.s4.intel.lua | 62 --------- reg_tests/chgres_cube/driver.s4.sh | 197 ---------------------------- reg_tests/global_cycle/driver.s4.sh | 84 ------------ reg_tests/grid_gen/driver.s4.sh | 109 --------------- reg_tests/ice_blend/driver.s4.sh | 66 ---------- reg_tests/rt.sh | 2 +- reg_tests/snow2mdl/driver.s4.sh | 72 ---------- sorc/machine-setup.sh | 8 -- 12 files changed, 9 insertions(+), 614 deletions(-) delete mode 100644 modulefiles/build.s4.intel.lua delete mode 100755 reg_tests/chgres_cube/driver.s4.sh delete mode 100755 reg_tests/global_cycle/driver.s4.sh delete mode 100755 reg_tests/grid_gen/driver.s4.sh delete mode 100755 reg_tests/ice_blend/driver.s4.sh delete mode 100755 reg_tests/snow2mdl/driver.s4.sh diff --git a/README.md b/README.md index 8e337e547..47347984c 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ It also uses the following repositories: ## Installing -On Orion, Hercules, Jet, Ursa, S4, Gaea c5, Gaea c6, and WCOSS2 do the following: +On Orion, Hercules, Jet, Ursa, Gaea c5, Gaea c6, and WCOSS2 do the following: 1) Set the 'fixed' directories using the `link_fixdirs.sh` script in `./fix`. Usage: `./link_fixdirs.sh $RUN_ENVIR $machine`, diff --git a/cmake/LibMPI.cmake b/cmake/LibMPI.cmake index 02278ef00..6630c5a2f 100644 --- a/cmake/LibMPI.cmake +++ b/cmake/LibMPI.cmake @@ -126,10 +126,6 @@ function (platform_name RETURN_VARIABLE) set (${RETURN_VARIABLE} "stampede" PARENT_SCOPE) - elseif (SITENAME MATCHES "^s4-submit.ssec.wisc.edu") - - set (${RETURN_VARIABLE} "s4" PARENT_SCOPE) - else () set (${RETURN_VARIABLE} "unknown" PARENT_SCOPE) diff --git a/docs/source/ufs_utils.rst b/docs/source/ufs_utils.rst index 2939e1d37..bec7a2230 100644 --- a/docs/source/ufs_utils.rst +++ b/docs/source/ufs_utils.rst @@ -682,7 +682,6 @@ The gdas_init utility is used to create coldstart initial conditions for global * Jet * WCOSS2 * Gaea C6 - * S4 (Only the chgres_cube step is supported, not the data pull step.) Location -------- @@ -698,7 +697,7 @@ Invoke the build script from the root directory: ./build_all.sh -Set the 'fixed' directories using the script in the './fix' subdirectory (where $MACHINE is "wcoss2", "ursa", "jet", "orion", "hercules", "s4", "gaeac5" or "gaeac6"): +Set the 'fixed' directories using the script in the './fix' subdirectory (where $MACHINE is "wcoss2", "ursa", "jet", "orion", "hercules", "gaeac5" or "gaeac6"): :: @@ -710,7 +709,7 @@ Configure for your experiment Edit the variables in the 'config' file for your experiment: * **EXTRACT_DIR** - Directory where data extracted from HPSS is stored. - * **EXTRACT_DATA** - Set to 'yes' to extract data from HPSS. If data has been extracted and is located in EXTRACT_DIR, set to 'no'. On 's4' this step can't be run. Instead, the data must be pulled from another machine. + * **EXTRACT_DATA** - Set to 'yes' to extract data from HPSS. If data has been extracted and is located in EXTRACT_DIR, set to 'no'. * **RUN_CHGRES** - To run chgres, set to 'yes'. To extract data only, set to 'no'. * **yy/mm/dd/hh** - The year/month/day/hour of your desired experiment. Use a four digit year and two digits for month/day/hour. **NOTE:** *The standard build of chgres_cube does NOT support experiments prior to June 12, 2019. To coldstart an experiment prior to these dates, contact a repository manager for assistance.* * **LEVS** - Number of hybrid levels plus 1. To run with 127 levels, set LEVS to 128. @@ -727,7 +726,7 @@ Note: This utility selects the ocean resolution in the set_fixed_files.sh script Kick off the utility -------------------- -Submit the driver script (where $MACHINE is 'gaeac6', 'ursa', 'jet', 'wcoss2', or 's4') +Submit the driver script (where $MACHINE is 'gaeac6', 'ursa', 'jet', 'wcoss2') :: diff --git a/fix/link_fixdirs.sh b/fix/link_fixdirs.sh index 14b30809e..debf75b7b 100755 --- a/fix/link_fixdirs.sh +++ b/fix/link_fixdirs.sh @@ -9,7 +9,7 @@ set -ex # 'nco' (copies data). # # $machine - is the machine. Choices are: -# 'wcoss2', 'ursa', 'jet', 'orion', 'hercules', 's4', 'gaeac5', 'gaeac6' +# 'wcoss2', 'ursa', 'jet', 'orion', 'hercules', 'gaeac5', 'gaeac6' RUN_ENVIR=${1} machine=${2} @@ -17,7 +17,7 @@ machine=${2} if [ $# -lt 2 ]; then set +x echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine' - echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | ursa | jet | orion | hercules | s4 | gaeac5 | gaeac6 )' + echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | ursa | jet | orion | hercules | gaeac5 | gaeac6 )' exit 1 fi @@ -28,10 +28,10 @@ if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then exit 1 fi -if [ $machine != wcoss2 -a $machine != ursa -a $machine != jet -a $machine != orion -a $machine != s4 -a $machine != hercules -a $machine != gaeac5 -a $machine != gaeac6 ]; then +if [ $machine != wcoss2 -a $machine != ursa -a $machine != jet -a $machine != orion -a $machine != hercules -a $machine != gaeac5 -a $machine != gaeac6 ]; then set +x echo '***ERROR*** unsupported machine' - echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | ursa | jet | orion | hercules | s4 | gaeac5 | gaeac6 )' + echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | ursa | jet | orion | hercules | gaeac5 | gaeac6 )' exit 1 fi @@ -52,8 +52,6 @@ elif [ $machine = "orion" -o $machine = "hercules" ]; then FIX_DIR="/work2/noaa/global/role-global/fix" elif [ $machine = "wcoss2" ]; then FIX_DIR="/lfs/h2/emc/global/noscrub/emc.global/FIX/fix" -elif [ $machine = "s4" ]; then - FIX_DIR="/data/prod/glopara/fix" elif [ $machine = "gaeac5" ]; then FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" elif [ $machine = "gaeac6" ]; then diff --git a/modulefiles/build.s4.intel.lua b/modulefiles/build.s4.intel.lua deleted file mode 100644 index 468647d26..000000000 --- a/modulefiles/build.s4.intel.lua +++ /dev/null @@ -1,62 +0,0 @@ -help([[ -Load environment to compile UFS_UTILS on S4 using Intel -]]) - -prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") - -stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" -load(pathJoin("stack-intel", hpc_intel_ver)) - -stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.0" -load(pathJoin("stack-intel-oneapi-mpi", impi_ver)) - -cmake_ver=os.getenv("cmake_ver") or "3.23.1" -load(pathJoin("cmake", cmake_ver)) - -bacio_ver=os.getenv("bacio_ver") or "2.4.1" -load(pathJoin("bacio", bacio_ver)) - -g2_ver=os.getenv("g2_ver") or "3.5.1" -load(pathJoin("g2", g2_ver)) - -ip_ver=os.getenv("ip_ver") or "4.3.0" -load(pathJoin("ip", ip_ver)) - -nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" -load(pathJoin("nemsio", nemsio_ver)) - -sp_ver=os.getenv("sp_ver") or "2.5.0" -load(pathJoin("sp", sp_ver)) - -w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" -load(pathJoin("w3emc", w3emc_ver)) - --- Uncomment when CHGRES_ALL is ON ---sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" ---load(pathJoin("sfcio", sfcio_ver)) - -sigio_ver=os.getenv("sigio_ver") or "2.3.2" -load(pathJoin("sigio", sigio_ver)) - -zlib_ver=os.getenv("zlib_ver") or "1.2.13" -load(pathJoin("zlib", zlib_ver)) - -png_ver=os.getenv("png_ver") or "1.6.37" -load(pathJoin("libpng", png_ver)) - -netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" -load(pathJoin("netcdf-c", netcdf_c_ver)) - -netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1" -load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) - -nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" -load(pathJoin("nccmp", nccmp_ver)) - -esmf_ver=os.getenv("esmf_ver") or "8.6.0" -load(pathJoin("esmf", esmf_ver)) - -nco_ver=os.getenv("nco_ver") or "5.0.6" -load(pathJoin("nco", nco_ver)) - -whatis("Description: UFS_UTILS build environment") diff --git a/reg_tests/chgres_cube/driver.s4.sh b/reg_tests/chgres_cube/driver.s4.sh deleted file mode 100755 index 86a68b77b..000000000 --- a/reg_tests/chgres_cube/driver.s4.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/bin/bash - -#----------------------------------------------------------------------------- -# -# Run the chgres_cube consistency tests on S4. -# -# Set WORK_DIR to a general working location outside the UFS_UTILS directory. -# The exact working directory (OUTDIR) will be WORK_DIR/reg_tests/chgres-cube. -# Set the PROJECT_CODE and QUEUE as appropriate. To see which projects you -# are authorized to use, type -# "sacctmgr show assoc Users= format=account,user,qos" -# -# Invoke the script with no arguments. A series of daisy-chained -# jobs will be submitted. To check the queue, type: -# "squeue -u USERNAME". -# -# The run output will be stored in OUTDIR. Log output from the suite -# will be in LOG_FILE. Once the suite has completed, a summary is -# placed in SUM_FILE. -# -# A test fails when its output does not match the baseline files as -# determined by the "nccmp" utility. The baseline files are stored in -# HOMEreg. -# -#----------------------------------------------------------------------------- - -set -x - -compiler=${compiler:-"intel"} - -source ../../sorc/machine-setup.sh > /dev/null 2>&1 -module use ../../modulefiles -module load build.$target.$compiler -module list - -export OUTDIR="${WORK_DIR:-/scratch/short/users/$LOGNAME}" -export OUTDIR="${OUTDIR}/reg-tests/chgres-cube" - -PROJECT_CODE="${PROJECT_CODE:-star}" -QUEUE="${QUEUE:-s4}" - -#----------------------------------------------------------------------------- -# Should not have to change anything below here. HOMEufs is the root -# directory of your UFS_UTILS clone. HOMEreg contains the input data -# and baseline data for each test. -#----------------------------------------------------------------------------- - -export UPDATE_BASELINE="FALSE" -#export UPDATE_BASELINE="TRUE" - -if [ "$UPDATE_BASELINE" = "TRUE" ]; then - source ../get_hash.sh -fi - -export HOMEufs=$PWD/../.. - -export HOMEreg=/data/users/dhuber/save/nems/role.ufsutils/ufs_utils/reg_tests/chgres_cube - -LOG_FILE=consistency.log -SUM_FILE=summary.log -rm -f $LOG_FILE* $SUM_FILE - -export OMP_STACKSIZE=1024M - -export APRUN=srun -export NCCMP=${NCCMP:-nccmp} -rm -fr $OUTDIR - -#----------------------------------------------------------------------------- -# Initialize C96 using FV3 warm restart files. -#----------------------------------------------------------------------------- - -LOG_FILE=consistency.log01 -export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST1=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.restart \ - -o $LOG_FILE -e $LOG_FILE ./c96.fv3.restart.sh) - -#----------------------------------------------------------------------------- -# Initialize C192 using FV3 tiled history files. -#----------------------------------------------------------------------------- - -LOG_FILE=consistency.log02 -export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST2=$(sbatch --parsable --ntasks-per-node=6 --nodes=2 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c192.fv3.history \ - -o $LOG_FILE -e $LOG_FILE ./c192.fv3.history.sh) - -#----------------------------------------------------------------------------- -# Initialize C96 using FV3 gaussian netcdf files. -#----------------------------------------------------------------------------- - -LOG_FILE=consistency.log03 -export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST3=$(sbatch --parsable --ntasks-per-node=12 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.netcdf \ - -o $LOG_FILE -e $LOG_FILE ./c96.fv3.netcdf.sh) - -#----------------------------------------------------------------------------- -# Initialize global C192 using GFS GRIB2 files. -#----------------------------------------------------------------------------- - -LOG_FILE=consistency.log04 -export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST4=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c192.gfs.grib2 \ - -o $LOG_FILE -e $LOG_FILE ./c192.gfs.grib2.sh) - -#----------------------------------------------------------------------------- -# Initialize CONUS 25-KM USING GFS GRIB2 files. -#----------------------------------------------------------------------------- - -LOG_FILE=consistency.log05 -export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST5=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 25km.conus.gfs.grib2.conus \ - -o $LOG_FILE -e $LOG_FILE ./25km.conus.gfs.grib2.sh) - -#----------------------------------------------------------------------------- -# Initialize CONUS 3-KM USING HRRR GRIB2 file WITH GFS PHYSICS. -#----------------------------------------------------------------------------- - -LOG_FILE=consistency.log06 -export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST6=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J 3km.conus.hrrr.gfssdf.grib2.conus \ - -o $LOG_FILE -e $LOG_FILE ./3km.conus.hrrr.gfssdf.grib2.sh) - -#----------------------------------------------------------------------------- -# Initialize CONUS 3-KM USING HRRR GRIB2 file WITH GSD PHYSICS AND SFC VARS FROM FILE. -#----------------------------------------------------------------------------- - -LOG_FILE=consistency.log07 -export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST7=$(sbatch --parsable --ntasks-per-node=6 --nodes=2 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J 3km.conus.hrrr.newsfc.grib2.conus \ - -o $LOG_FILE -e $LOG_FILE ./3km.conus.hrrr.newsfc.grib2.sh) - -#----------------------------------------------------------------------------- -# Initialize CONUS 13-KM USING NAM GRIB2 file WITH GFS PHYSICS . -#----------------------------------------------------------------------------- - -LOG_FILE=consistency.log08 -export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST8=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 13km.conus.nam.grib2.conus \ - -o $LOG_FILE -e $LOG_FILE ./13km.conus.nam.grib2.sh) - -#----------------------------------------------------------------------------- -# Initialize CONUS 13-KM USING RAP GRIB2 file WITH GSD PHYSICS . -#----------------------------------------------------------------------------- - -LOG_FILE=consistency.log09 -export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST9=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 13km.conus.rap.grib2.conus \ - -o $LOG_FILE -e $LOG_FILE ./13km.conus.rap.grib2.sh) - -#----------------------------------------------------------------------------- -# Initialize CONUS 13-KM NA USING NCEI GFS GRIB2 file WITH GFS PHYSICS . -#----------------------------------------------------------------------------- - -LOG_FILE=consistency.log10 -export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST10=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 13km.na.gfs.ncei.grib2.conus \ - -o $LOG_FILE -e $LOG_FILE ./13km.na.gfs.ncei.grib2.sh) - -#----------------------------------------------------------------------------- -# Initialize C96 WAM IC using FV3 gaussian netcdf files. -#----------------------------------------------------------------------------- - -LOG_FILE=consistency.log11 -export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST11=$(sbatch --parsable --ntasks-per-node=12 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.netcdf2wam \ - -o $LOG_FILE -e $LOG_FILE ./c96.fv3.netcdf2wam.sh) - -#----------------------------------------------------------------------------- -# Initialize CONUS 25-KM USING GFS PGRIB2+BGRIB2 files. -#----------------------------------------------------------------------------- - -LOG_FILE=consistency.log12 -export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST12=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 25km.conus.gfs.pbgrib2.conus \ - -o $LOG_FILE -e $LOG_FILE ./25km.conus.gfs.pbgrib2.sh) - -#----------------------------------------------------------------------------- -# Initialize C96 using GEFS GRIB2 file. -#----------------------------------------------------------------------------- - -LOG_FILE=consistency.log13 -export OMP_NUM_THREADS=1 # should match cpus-per-task -TEST13=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c96.gefs.grib2 \ - -o $LOG_FILE -e $LOG_FILE ./c96.gefs.grib2.sh) - -#----------------------------------------------------------------------------- -# Create summary log. -#----------------------------------------------------------------------------- -LOG_FILE=consistency.log -sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \ - --open-mode=append -q $QUEUE -d\ - afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5:$TEST6:$TEST7:$TEST8:$TEST9:$TEST10:$TEST11:$TEST12:$TEST13 << EOF -#!/bin/bash -grep -a '<<<' $LOG_FILE* > $SUM_FILE -EOF - -exit 0 diff --git a/reg_tests/global_cycle/driver.s4.sh b/reg_tests/global_cycle/driver.s4.sh deleted file mode 100755 index 439925c7e..000000000 --- a/reg_tests/global_cycle/driver.s4.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -#----------------------------------------------------------------------------- -# -# Run global_cycle consistency test on S4. -# -# Set $WORK_DIR to your working directory. Set the project code -# and queue as appropriate. -# -# Invoke the script from the command line as follows: ./$script -# -# Log output is placed in consistency.log??. A summary is -# placed in summary.log -# -# A test fails when its output does not match the baseline files -# as determined by the 'nccmp' utility. This baseline files are -# stored in HOMEreg. -# -#----------------------------------------------------------------------------- - -set -x - -compiler=${compiler:-"intel"} - -source ../../sorc/machine-setup.sh > /dev/null 2>&1 -module use ../../modulefiles -module load build.$target.$compiler -module list - -WORK_DIR="${WORK_DIR:-/scratch/short/users/$LOGNAME}" - -PROJECT_CODE="${PROJECT_CODE:-star}" -QUEUE="${QUEUE:-batch}" - -#----------------------------------------------------------------------------- -# Should not have to change anything below. -#----------------------------------------------------------------------------- - -export UPDATE_BASELINE="FALSE" -#export UPDATE_BASELINE="TRUE" - -if [ "$UPDATE_BASELINE" = "TRUE" ]; then - source ../get_hash.sh -fi - -DATA_DIR="${WORK_DIR}/reg-tests/global-cycle" - -export HOMEreg=/data/users/dhuber/save/nems/role.ufsutils/ufs_utils/reg_tests/global_cycle - -export OMP_NUM_THREADS_CY=2 - -export APRUNCY="srun" - -export NWPROD=$PWD/../.. - -reg_dir=$PWD - -LOG_FILE=consistency.log01 -export DATA="${DATA_DIR}/test1" -export COMOUT=$DATA -TEST1=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c768.fv3gfs \ - -o $LOG_FILE -e $LOG_FILE ./C768.fv3gfs.sh) - -LOG_FILE=consistency.log02 -export DATA="${DATA_DIR}/test2" -export COMOUT=$DATA -TEST2=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c768.lndincsoil \ - -o $LOG_FILE -e $LOG_FILE ./C768.lndincsoil.sh) - -LOG_FILE=consistency.log03 -export DATA="${DATA_DIR}/test3" -export COMOUT=$DATA -TEST3=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c768.lndincsnow \ - -o $LOG_FILE -e $LOG_FILE ./C768.lndincsnow.sh) - -LOG_FILE=consistency.log -sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J summary -o $LOG_FILE -e $LOG_FILE \ - --open-mode=append -q $QUEUE -d\ - afterok:$TEST1:$TEST2:$TEST3 << EOF -#!/bin/bash -grep -a '<<<' ${LOG_FILE}* > summary.log -EOF - -exit diff --git a/reg_tests/grid_gen/driver.s4.sh b/reg_tests/grid_gen/driver.s4.sh deleted file mode 100755 index 190c22c9c..000000000 --- a/reg_tests/grid_gen/driver.s4.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/bash - -#----------------------------------------------------------------------------- -# -# Run grid generation consistency tests on S4. -# -# Set WORK_DIR to your working directory. Set the PROJECT_CODE and QUEUE -# as appropriate. To see which projects you are authorized to use, -# type -# "sacctmgr show assoc Users= format=account,user,qos" -# -# Invoke the script with no arguments. A series of daily- -# chained jobs will be submitted. To check the queue, type: -# "squeue -u USERNAME". -# -# Log output from the suite will be in LOG_FILE. Once the suite -# has completed, a summary is placed in SUM_FILE. -# -# A test fails when its output does not match the baseline files as -# determined by the "nccmp" utility. The baseline files are stored in -# HOMEreg -# -#----------------------------------------------------------------------------- - -compiler=${compiler:-"intel"} - -source ../../sorc/machine-setup.sh > /dev/null 2>&1 -module use ../../modulefiles -module load build.$target.$compiler -module list - -set -x - -export WORK_DIR="${WORK_DIR:-/scratch/short/users/$LOGNAME}" -export WORK_DIR="${WORK_DIR}/reg-tests/grid-gen" -QUEUE="${QUEUE:-s4}" -PROJECT_CODE="${PROJECT_CODE:-star}" - -#----------------------------------------------------------------------------- -# Should not have to change anything below here. -#----------------------------------------------------------------------------- - -export UPDATE_BASELINE="FALSE" -#export UPDATE_BASELINE="TRUE" - -if [ "$UPDATE_BASELINE" = "TRUE" ]; then - source ../get_hash.sh -fi - -LOG_FILE=consistency.log -SUM_FILE=summary.log -export home_dir=$PWD/../.. -export APRUN=time -export APRUN_SFC=srun -export OMP_STACKSIZE=2048m -export machine=S4 -export HOMEreg=/data/users/dhuber/save/nems/role.ufsutils/ufs_utils/reg_tests/grid_gen/baseline_data - -ulimit -a -#ulimit -s unlimited - -rm -fr $WORK_DIR - -export OMP_NUM_THREADS=24 - -#----------------------------------------------------------------------------- -# C96 uniform grid -#----------------------------------------------------------------------------- - -TEST1=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.uniform \ - -o $LOG_FILE -e $LOG_FILE ./c96.uniform.sh) - -#----------------------------------------------------------------------------- -# C96 uniform grid using viirs vegetation data. -#----------------------------------------------------------------------------- - -TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.viirs.vegt \ - -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST1 ./c96.viirs.vegt.sh) - -#----------------------------------------------------------------------------- -# gfdl regional grid -#----------------------------------------------------------------------------- - -TEST3=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J gfdl.regional \ - -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST2 ./gfdl.regional.sh) - -#----------------------------------------------------------------------------- -# esg regional grid -#----------------------------------------------------------------------------- - -TEST4=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J esg.regional \ - -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST3 ./esg.regional.sh) - -#----------------------------------------------------------------------------- -# Regional GSL gravity wave drag test. -#----------------------------------------------------------------------------- - -TEST5=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J reg.gsl.gwd \ - -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST4 ./regional.gsl.gwd.sh) - -#----------------------------------------------------------------------------- -# Create summary log. -#----------------------------------------------------------------------------- - -sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J grid_summary -o $LOG_FILE -e $LOG_FILE \ - --open-mode=append -q $QUEUE -d afterok:$TEST5 << EOF -#!/bin/bash -grep -a '<<<' $LOG_FILE > $SUM_FILE -EOF diff --git a/reg_tests/ice_blend/driver.s4.sh b/reg_tests/ice_blend/driver.s4.sh deleted file mode 100755 index 7aee14a02..000000000 --- a/reg_tests/ice_blend/driver.s4.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -#----------------------------------------------------------------------------- -# -# Run ice_blend consistency test on S4. -# -# Set $DATA to your working directory. Set the project code (SBATCH -A) -# and queue (SBATCH -q) as appropriate. -# -# Invoke the script as follows: sbatch $script -# -# Log output is placed in consistency.log. A summary is -# placed in summary.log -# -# The test fails when its output does not match the baseline file -# as determined by the 'cmp' command. The baseline file is -# stored in HOMEreg. -# -#----------------------------------------------------------------------------- - -#SBATCH -J ice_blend -#SBATCH -A s4 -#SBATCH --open-mode=truncate -#SBATCH -o consistency.log -#SBATCH -e consistency.log -#SBATCH --ntasks=1 -#SBATCH -q s4 -#SBATCH -t 00:03:00 - -set -x - -compiler=${compiler:-"intel"} - -source ../../sorc/machine-setup.sh > /dev/null 2>&1 -module use ../../modulefiles -module load build.$target.$compiler -module list - -export DATA="${WORK_DIR:-/scratch/short/users/$LOGNAME}" -export DATA="${DATA}/reg-tests/ice-blend" - -#----------------------------------------------------------------------------- -# Should not have to change anything below. -#----------------------------------------------------------------------------- - -export UPDATE_BASELINE="FALSE" -#export UPDATE_BASELINE="TRUE" - -if [ "$UPDATE_BASELINE" = "TRUE" ]; then - source ../get_hash.sh -fi - -export WGRIB=/data/prod/hpc-stack/intel-2022.1/grib_util/1.2.2/bin/wgrib -export WGRIB2=/data/prod/hpc-stack/intel-2022.1/wgrib2/2.0.8/bin/wgrib2 -export COPYGB=/data/prod/hpc-stack/intel-2022.1/grib_util/1.2.2/bin/copygb -export COPYGB2=/data/prod/hpc-stack/intel-2022.1/grib_util/1.2.2/bin/copygb2 -export CNVGRIB=/data/prod/hpc-stack/intel-2022.1/grib_util/1.2.2/bin/cnvgrib - -export HOMEreg=/data/users/dhuber/save/nems/role.ufsutils/ufs_utils/reg_tests/ice_blend -export HOMEgfs=$PWD/../.. - -rm -fr $DATA - -./ice_blend.sh - -exit 0 diff --git a/reg_tests/rt.sh b/reg_tests/rt.sh index 12b7ce865..4b5f31319 100755 --- a/reg_tests/rt.sh +++ b/reg_tests/rt.sh @@ -127,7 +127,7 @@ done for dir in weight_gen ice_blend; do cd $dir - if [[ $target == "ursa" ]] || [[ $target == "jet" ]] || [[ $target == "orion" ]] || [[ $target == "s4" ]] || [[ $target == "hercules" ]] ; then + if [[ $target == "ursa" ]] || [[ $target == "jet" ]] || [[ $target == "orion" ]] || [[ $target == "hercules" ]] ; then sbatch -A ${PROJECT_CODE} ./driver.$target.sh elif [[ $target == "wcoss2" ]] ; then qsub -v WORK_DIR ./driver.$target.sh diff --git a/reg_tests/snow2mdl/driver.s4.sh b/reg_tests/snow2mdl/driver.s4.sh deleted file mode 100755 index 608a19584..000000000 --- a/reg_tests/snow2mdl/driver.s4.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -#----------------------------------------------------------------------------- -# -# Run snow2mdl consistency test on S4. -# -# Set $DATA_ROOT to your working directory. Set the project code -# and queue as appropriate. -# -# Invoke the script as follows: ./$script -# -# Log output is placed in consistency.log. A summary is -# placed in summary.log -# -# The test fails when its output does not match the baseline file -# as determined by the 'cmp' command. The baseline file is -# stored in HOMEreg. -# -#----------------------------------------------------------------------------- - -set -x - -compiler=${compiler:-"intel"} - -source ../../sorc/machine-setup.sh > /dev/null 2>&1 -module use ../../modulefiles -module load build.$target.$compiler -module list - -DATA_ROOT="${WORK_DIR:-/scratch/short/users/$LOGNAME}" -DATA_ROOT="${DATA_ROOT}/reg-tests/snow2mdl" - -PROJECT_CODE="${PROJECT_CODE:-star}" -QUEUE="${QUEUE:-s4}" - -#----------------------------------------------------------------------------- -# Should not have to change anything below. -#----------------------------------------------------------------------------- - -export UPDATE_BASELINE="FALSE" -#export UPDATE_BASELINE="TRUE" - -if [ "$UPDATE_BASELINE" = "TRUE" ]; then - source ../get_hash.sh -fi - -export HOMEreg=/data/users/dhuber/save/nems/role.ufsutils/ufs_utils/reg_tests/snow2mdl -export HOMEgfs=$PWD/../.. -export WGRIB=/data/prod/hpc-stack/intel-2022.1/grib_util/1.2.2/bin/wgrib -export WGRIB2=/data/prod/hpc-stack/intel-2022.1/wgrib2/2.0.8/bin/wgrib2 - -# The first test mimics GFS OPS. - -export DATA="${DATA_ROOT}/test.ops" -TEST1=$(sbatch --parsable -J snow.ops -A ${PROJECT_CODE} -o consistency.log -e consistency.log \ - --ntasks=1 -q ${QUEUE} -t 00:03:00 ./snow2mdl.ops.sh) - -# The second test is for the new AFWA global GRIB2 data. - -export DATA="${DATA_ROOT}/test.global" -TEST2=$(sbatch --parsable -J snow.global -A ${PROJECT_CODE} -o consistency.log -e consistency.log \ - --ntasks=1 -q ${QUEUE} -t 00:03:00 -d afterok:$TEST1 ./snow2mdl.global.sh) - -# Create summary file. - -sbatch --nodes=1 -t 0:01:00 -A ${PROJECT_CODE} -J snow_summary -o consistency.log -e consistency.log \ - --open-mode=append -q ${QUEUE} -d afterok:$TEST2 << EOF -#!/bin/bash -grep -a '<<<' consistency.log > summary.log -EOF - -exit 0 diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index 3d712ca4d..0dc85566c 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -70,14 +70,6 @@ elif [[ "$(hostname)" =~ "hercules" || "$(hostname)" =~ "Hercules" ]]; then elif [[ -d /work/00315 && -d /scratch/00315 ]] ; then target=stampede module purge -elif [[ -d /data/prod ]] ; then - # We are on SSEC S4 - if ( ! eval module help > /dev/null 2>&1 ) ; then - echo load the module command 1>&2 - source /usr/share/lmod/lmod/init/$__ms_shell - fi - target=s4 - module purge else if [[ ! -v PW_CSP ]]; then set +x From 852e44c75e18513cbc1aff11476c3a63c4d1f3d3 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 13 Jun 2025 18:12:57 +0000 Subject: [PATCH 31/54] Remove references to GaeaC5. Fixes #1064. --- README.md | 2 +- docs/source/ufs_utils.rst | 2 +- fix/link_fixdirs.sh | 10 ++--- modulefiles/build.gaeac5.intel.lua | 66 ------------------------------ sorc/machine-setup.sh | 12 ------ 5 files changed, 6 insertions(+), 86 deletions(-) delete mode 100644 modulefiles/build.gaeac5.intel.lua diff --git a/README.md b/README.md index 47347984c..e560dcaa7 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ It also uses the following repositories: ## Installing -On Orion, Hercules, Jet, Ursa, Gaea c5, Gaea c6, and WCOSS2 do the following: +On Orion, Hercules, Jet, Ursa, Gaea c6, and WCOSS2 do the following: 1) Set the 'fixed' directories using the `link_fixdirs.sh` script in `./fix`. Usage: `./link_fixdirs.sh $RUN_ENVIR $machine`, diff --git a/docs/source/ufs_utils.rst b/docs/source/ufs_utils.rst index bec7a2230..6e872fba2 100644 --- a/docs/source/ufs_utils.rst +++ b/docs/source/ufs_utils.rst @@ -697,7 +697,7 @@ Invoke the build script from the root directory: ./build_all.sh -Set the 'fixed' directories using the script in the './fix' subdirectory (where $MACHINE is "wcoss2", "ursa", "jet", "orion", "hercules", "gaeac5" or "gaeac6"): +Set the 'fixed' directories using the script in the './fix' subdirectory (where $MACHINE is "wcoss2", "ursa", "jet", "orion", "hercules", or "gaeac6"): :: diff --git a/fix/link_fixdirs.sh b/fix/link_fixdirs.sh index debf75b7b..095a94347 100755 --- a/fix/link_fixdirs.sh +++ b/fix/link_fixdirs.sh @@ -9,7 +9,7 @@ set -ex # 'nco' (copies data). # # $machine - is the machine. Choices are: -# 'wcoss2', 'ursa', 'jet', 'orion', 'hercules', 'gaeac5', 'gaeac6' +# 'wcoss2', 'ursa', 'jet', 'orion', 'hercules', 'gaeac6' RUN_ENVIR=${1} machine=${2} @@ -17,7 +17,7 @@ machine=${2} if [ $# -lt 2 ]; then set +x echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine' - echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | ursa | jet | orion | hercules | gaeac5 | gaeac6 )' + echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | ursa | jet | orion | hercules | gaeac6 )' exit 1 fi @@ -28,10 +28,10 @@ if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then exit 1 fi -if [ $machine != wcoss2 -a $machine != ursa -a $machine != jet -a $machine != orion -a $machine != hercules -a $machine != gaeac5 -a $machine != gaeac6 ]; then +if [ $machine != wcoss2 -a $machine != ursa -a $machine != jet -a $machine != orion -a $machine != hercules -a $machine != gaeac6 ]; then set +x echo '***ERROR*** unsupported machine' - echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | ursa | jet | orion | hercules | gaeac5 | gaeac6 )' + echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | ursa | jet | orion | hercules | gaeac6 )' exit 1 fi @@ -52,8 +52,6 @@ elif [ $machine = "orion" -o $machine = "hercules" ]; then FIX_DIR="/work2/noaa/global/role-global/fix" elif [ $machine = "wcoss2" ]; then FIX_DIR="/lfs/h2/emc/global/noscrub/emc.global/FIX/fix" -elif [ $machine = "gaeac5" ]; then - FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" elif [ $machine = "gaeac6" ]; then FIX_DIR="/gpfs/f6/drsa-precip3/world-shared/role.glopara/fix" fi diff --git a/modulefiles/build.gaeac5.intel.lua b/modulefiles/build.gaeac5.intel.lua deleted file mode 100644 index b2132f3fd..000000000 --- a/modulefiles/build.gaeac5.intel.lua +++ /dev/null @@ -1,66 +0,0 @@ -help([[ -Load environment to compile UFS_UTILS on Gaea using Intel -]]) - -prepend_path("MODULEPATH", "/usw/hpss/modulefiles") -load("hsi") - -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") - -stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" -load(pathJoin("stack-intel", stack_intel_ver)) - -stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28" -load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) - -cmake_ver=os.getenv("cmake_ver") or "3.23.1" -load(pathJoin("cmake", cmake_ver)) - -bacio_ver=os.getenv("bacio_ver") or "2.4.1" -load(pathJoin("bacio", bacio_ver)) - -g2_ver=os.getenv("g2_ver") or "3.5.1" -load(pathJoin("g2", g2_ver)) - -ip_ver=os.getenv("ip_ver") or "4.3.0" -load(pathJoin("ip", ip_ver)) - -nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" -load(pathJoin("nemsio", nemsio_ver)) - -sp_ver=os.getenv("sp_ver") or "2.5.0" -load(pathJoin("sp", sp_ver)) - -w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" -load(pathJoin("w3emc", w3emc_ver)) - --- Uncomment when CHGRES_ALL is ON ---sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" ---load(pathJoin("sfcio", sfcio_ver)) - -sigio_ver=os.getenv("sigio_ver") or "2.3.2" -load(pathJoin("sigio", sigio_ver)) - -zlib_ver=os.getenv("zlib_ver") or "1.2.13" -load(pathJoin("zlib", zlib_ver)) - -png_ver=os.getenv("png_ver") or "1.6.37" -load(pathJoin("libpng", png_ver)) - -netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" -load(pathJoin("netcdf-c", netcdf_c_ver)) - -netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1" -load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) - -nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" -load(pathJoin("nccmp", nccmp_ver)) - -esmf_ver=os.getenv("esmf_ver") or "8.6.0" -load(pathJoin("esmf", esmf_ver)) - -nco_ver=os.getenv("nco_ver") or "5.0.6" -load(pathJoin("nco", nco_ver)) - -whatis("Description: UFS_UTILS build environment") - diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index 0dc85566c..55daa6496 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -46,18 +46,6 @@ elif [[ -d /scratch3 ]] ; then fi target=ursa module purge -elif [[ "$(hostname)" == "gaea5"* || "$(hostname)" =~ c5n[0-9]+ ]] && [[ -d /gpfs/f5 ]] ; then - # We are on GAEAC5. - if ( ! eval module help > /dev/null 2>&1 ) ; then - # We cannot simply load the module command. The GAEA - # /etc/profile modifies a number of module-related variables - # before loading the module command. Without those variables, - # the module command fails. Hence we actually have to source - # /etc/profile here. - source /etc/profile - fi - module reset - target=gaeac5 elif [[ "$(hostname)" == "gaea6"* || "$(hostname)" =~ c6n[0-9]+ ]] && [[ -d /gpfs/f6 ]] ; then target=gaeac6 source /opt/cray/pe/lmod/8.7.31/init/$__ms_shell From fb6adf37bf34e3dbfb45112e5c12db05e421c0c1 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 13 Jun 2025 20:43:31 +0000 Subject: [PATCH 32/54] Load ip v5 AND sp so I don't need to modify sfcsub.F. Fixes #1064. --- CMakeLists.txt | 7 +++++-- modulefiles/build.ursa.intelllvm.lua | 4 ++-- sorc/chgres_cube.fd/CMakeLists.txt | 4 ---- sorc/chgres_cube.fd/model_grid.F90 | 3 --- sorc/global_cycle.fd/CMakeLists.txt | 4 ---- sorc/global_cycle.fd/cycle.F90 | 3 --- sorc/global_cycle.fd/land_increments.F90 | 3 --- sorc/weight_gen.fd/CMakeLists.txt | 3 --- sorc/weight_gen.fd/scrip.F90 | 4 ---- 9 files changed, 7 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a27d2ccb..170d73e4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,9 +119,12 @@ if(CHGRES_ALL OR GBLEVENTS) find_package(sigio 2.3.0 REQUIRED) endif() find_package(ip 3.3.3 REQUIRED) -if(ip_VERSION LESS 5.0) +# Can't use ip v5 without sp because modfications would need to be +# made to global_cycle routine sfcsub.F, which is part of +# ccpp-physics repo. +#if(ip_VERSION LESS 5.0) find_package(sp 2.3.3 REQUIRED) -endif() +#endif() find_package(g2 3.4.3 REQUIRED) # If doxygen documentation we enabled, build it. This must come before diff --git a/modulefiles/build.ursa.intelllvm.lua b/modulefiles/build.ursa.intelllvm.lua index 8df2bde4c..618b1d18a 100644 --- a/modulefiles/build.ursa.intelllvm.lua +++ b/modulefiles/build.ursa.intelllvm.lua @@ -25,8 +25,8 @@ load(pathJoin("ip", ip_ver)) nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" load(pathJoin("nemsio", nemsio_ver)) ---sp_ver=os.getenv("sp_ver") or "2.5.0" ---load(pathJoin("sp", sp_ver)) +sp_ver=os.getenv("sp_ver") or "2.5.0" +load(pathJoin("sp", sp_ver)) w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" load(pathJoin("w3emc", w3emc_ver)) diff --git a/sorc/chgres_cube.fd/CMakeLists.txt b/sorc/chgres_cube.fd/CMakeLists.txt index 70d7dc13c..4ff86450f 100644 --- a/sorc/chgres_cube.fd/CMakeLists.txt +++ b/sorc/chgres_cube.fd/CMakeLists.txt @@ -31,10 +31,6 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ffree-line-length-0 -fdefault-real-8") endif() -if(ip_VERSION GREATER_EQUAL 5.0.0) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DIP_V5") -endif() - if(CHGRES_ALL) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DCHGRES_ALL") endif() diff --git a/sorc/chgres_cube.fd/model_grid.F90 b/sorc/chgres_cube.fd/model_grid.F90 index 60d79479b..74f4c341d 100644 --- a/sorc/chgres_cube.fd/model_grid.F90 +++ b/sorc/chgres_cube.fd/model_grid.F90 @@ -160,9 +160,6 @@ subroutine define_input_grid_gaussian(npets) #ifdef CHGRES_ALL use sfcio_module use sigio_module -#endif -#ifdef IP_V5 - use sp_mod #endif use netcdf diff --git a/sorc/global_cycle.fd/CMakeLists.txt b/sorc/global_cycle.fd/CMakeLists.txt index 7a2da4ed1..8f5098dea 100644 --- a/sorc/global_cycle.fd/CMakeLists.txt +++ b/sorc/global_cycle.fd/CMakeLists.txt @@ -22,10 +22,6 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$") endif() endif() -if(ip_VERSION GREATER_EQUAL 5.0.0) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DIP_V5") -endif() - set(exe_name global_cycle) add_library(global_cycle_lib STATIC ${lib_src}) diff --git a/sorc/global_cycle.fd/cycle.F90 b/sorc/global_cycle.fd/cycle.F90 index 493888d7d..6431dca62 100644 --- a/sorc/global_cycle.fd/cycle.F90 +++ b/sorc/global_cycle.fd/cycle.F90 @@ -924,9 +924,6 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& USE UTILS USE GDSWZD_MOD -#ifdef IP_V5 - use sp_mod -#endif USE READ_WRITE_DATA, ONLY : IDIM_GAUS, JDIM_GAUS, & SLMSK_GAUS, DTREF_GAUS, & NSST_DATA diff --git a/sorc/global_cycle.fd/land_increments.F90 b/sorc/global_cycle.fd/land_increments.F90 index 6efae21a8..0d7297b59 100644 --- a/sorc/global_cycle.fd/land_increments.F90 +++ b/sorc/global_cycle.fd/land_increments.F90 @@ -48,9 +48,6 @@ subroutine gaussian_to_fv3_interp(lsoil_incr,rla,rlo, & use utils use gdswzd_mod -#ifdef IP_V5 - use sp_mod -#endif use read_write_data, only : idim_gaus, jdim_gaus, & stc_inc_gaus, soilsnow_gaus, slc_inc_gaus use mpi diff --git a/sorc/weight_gen.fd/CMakeLists.txt b/sorc/weight_gen.fd/CMakeLists.txt index 3890aa6d7..4c408d441 100644 --- a/sorc/weight_gen.fd/CMakeLists.txt +++ b/sorc/weight_gen.fd/CMakeLists.txt @@ -7,9 +7,6 @@ if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel|IntelLLVM)$") elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8") endif() -if(ip_VERSION GREATER_EQUAL 5.0.0) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DIP_V5") -endif() set(exe_name weight_gen) add_executable(${exe_name} ${fortran_src}) diff --git a/sorc/weight_gen.fd/scrip.F90 b/sorc/weight_gen.fd/scrip.F90 index 0ff0ac3a7..839767dfc 100644 --- a/sorc/weight_gen.fd/scrip.F90 +++ b/sorc/weight_gen.fd/scrip.F90 @@ -18,10 +18,6 @@ !! @author George Gayno NOAA/EMC program scrip -#ifdef IP_V5 - use sp_mod -#endif - implicit none character(len=128) :: outfile From d98c8680ec945811d5257b93a368e9d741525def Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Wed, 18 Jun 2025 13:32:35 +0000 Subject: [PATCH 33/54] Fix typo in regression test driver script. Fixes #1064. --- reg_tests/rt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reg_tests/rt.sh b/reg_tests/rt.sh index 4b5f31319..04480bbd9 100755 --- a/reg_tests/rt.sh +++ b/reg_tests/rt.sh @@ -44,7 +44,7 @@ else elif [[ -d /lfs/h1 ]] ; then target=WCOSS2 elif [[ -d /scratch3 ]] ; then - target=Usra + target=Ursa fi echo "Clone Failed" | mail -s "UFS_UTILS Consistency Tests failed on ${target}" ${MAILTO} fi From ecfced374eb1385eb12cf73f8678bb9be62d5d80 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Wed, 18 Jun 2025 14:41:54 +0000 Subject: [PATCH 34/54] Clean up regression test driver script. Fixes #1064. --- reg_tests/rt.sh | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/reg_tests/rt.sh b/reg_tests/rt.sh index 04480bbd9..0d008a60d 100755 --- a/reg_tests/rt.sh +++ b/reg_tests/rt.sh @@ -88,34 +88,29 @@ cd ../reg_tests set -x -#if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hercules" ]] || [[ $target == "wcoss2" ]] ; then -#if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hercules" ]] ; then +cd regrid_sfc +./driver.sh - cd regrid_sfc - ./driver.sh +wait_for_fin - wait_for_fin +cd .. - cd .. +export ACCOUNT=$PROJECT_CODE +export STMP=$WORK_DIR/reg-tests - export ACCOUNT=$PROJECT_CODE - export STMP=$WORK_DIR/reg-tests +cd ocnice_prep +./rt.sh - cd ocnice_prep - ./rt.sh +wait_for_fin - wait_for_fin +cd .. - cd .. +cd cpld_gridgen +./rt.sh - cd cpld_gridgen - ./rt.sh +wait_for_fin - wait_for_fin - - cd .. - -#fi +cd .. for dir in snow2mdl global_cycle chgres_cube grid_gen; do cd $dir From 84f78f7a9eef46c8b6166d3930dc8138f33a5d8d Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Wed, 18 Jun 2025 14:59:20 +0000 Subject: [PATCH 35/54] Update comments in CMakelists.txt concerning use of sp and ip v5+ Fixes #1064. --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 170d73e4f..3008ff043 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,9 +119,9 @@ if(CHGRES_ALL OR GBLEVENTS) find_package(sigio 2.3.0 REQUIRED) endif() find_package(ip 3.3.3 REQUIRED) -# Can't use ip v5 without sp because modfications would need to be -# made to global_cycle routine sfcsub.F, which is part of -# ccpp-physics repo. +# TODO: Disable sp when using ip v5.0+. +# This requires an update to ccpp-physics and global_cycle's +# sfcsub.F module. #if(ip_VERSION LESS 5.0) find_package(sp 2.3.3 REQUIRED) #endif() From 9795324524089f15c8c9ca66a7be091d35474188 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 20 Jun 2025 13:38:25 -0500 Subject: [PATCH 36/54] Upgrade to spack-stack 1.9.1 on RDHPCS --- modulefiles/build.gaeac6.intel.lua | 16 ++++++++-------- modulefiles/build.hercules.intel.lua | 24 +++++++++++++++--------- modulefiles/build.hercules.intelllvm.lua | 22 +++++++++++----------- modulefiles/build.orion.intel.lua | 24 +++++++++++++++--------- modulefiles/build.orion.intelllvm.lua | 22 +++++++++++----------- 5 files changed, 60 insertions(+), 48 deletions(-) diff --git a/modulefiles/build.gaeac6.intel.lua b/modulefiles/build.gaeac6.intel.lua index bc0064518..09cd86031 100644 --- a/modulefiles/build.gaeac6.intel.lua +++ b/modulefiles/build.gaeac6.intel.lua @@ -5,15 +5,15 @@ Load environment to compile UFS_UTILS on Gaea C6 using Intel prepend_path("MODULEPATH", "/usw/hpss/modulefiles") load("hsi") -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.9.1/envs/ue-intel-2023.2.0/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" load(pathJoin("stack-intel", stack_intel_ver)) -stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29" +stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.30" load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) -cmake_ver=os.getenv("cmake_ver") or "3.23.1" +cmake_ver=os.getenv("cmake_ver") or "3.27.9" load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" @@ -22,7 +22,7 @@ load(pathJoin("bacio", bacio_ver)) g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) -ip_ver=os.getenv("ip_ver") or "4.3.0" +ip_ver=os.getenv("ip_ver") or "5.1.0" load(pathJoin("ip", ip_ver)) nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" @@ -35,10 +35,10 @@ w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" load(pathJoin("w3emc", w3emc_ver)) -- Uncomment when CHGRES_ALL is ON ---sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" +--sfcio_ver=os.getenv("sfcio_ver") or "1.4.2" --load(pathJoin("sfcio", sfcio_ver)) -sigio_ver=os.getenv("sigio_ver") or "2.3.2" +sigio_ver=os.getenv("sigio_ver") or "2.3.3" load(pathJoin("sigio", sigio_ver)) zlib_ver=os.getenv("zlib_ver") or "1.2.13" @@ -56,10 +56,10 @@ load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" load(pathJoin("nccmp", nccmp_ver)) -esmf_ver=os.getenv("esmf_ver") or "8.6.0" +esmf_ver=os.getenv("esmf_ver") or "8.8.0" load(pathJoin("esmf", esmf_ver)) -nco_ver=os.getenv("nco_ver") or "5.0.6" +nco_ver=os.getenv("nco_ver") or "5.2.4" load(pathJoin("nco", nco_ver)) whatis("Description: UFS_UTILS build environment") diff --git a/modulefiles/build.hercules.intel.lua b/modulefiles/build.hercules.intel.lua index 277909359..7e5d777b2 100644 --- a/modulefiles/build.hercules.intel.lua +++ b/modulefiles/build.hercules.intel.lua @@ -2,15 +2,15 @@ help([[ Load environment to compile UFS_UTILS on Hercules using Intel Classic ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") +prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") -stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" +stack_intel_ver=os.getenv("stack_intel_ver") or "2024.2.1" load(pathJoin("stack-intel", stack_intel_ver)) -stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0" +stack_impi_ver=os.getenv("stack_impi_ver") or "2021.13" load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) -cmake_ver=os.getenv("cmake_ver") or "3.23.1" +cmake_ver=os.getenv("cmake_ver") or "3.27.9" load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" @@ -19,7 +19,7 @@ load(pathJoin("bacio", bacio_ver)) g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) -ip_ver=os.getenv("ip_ver") or "4.3.0" +ip_ver=os.getenv("ip_ver") or "5.1.0" load(pathJoin("ip", ip_ver)) nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" @@ -32,10 +32,10 @@ w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" load(pathJoin("w3emc", w3emc_ver)) -- Uncomment when CHGRES_ALL is ON ---sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" +--sfcio_ver=os.getenv("sfcio_ver") or "1.4.2" --load(pathJoin("sfcio", sfcio_ver)) -sigio_ver=os.getenv("sigio_ver") or "2.3.2" +sigio_ver=os.getenv("sigio_ver") or "2.3.3" load(pathJoin("sigio", sigio_ver)) zlib_ver=os.getenv("zlib_ver") or "1.2.13" @@ -53,10 +53,16 @@ load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" load(pathJoin("nccmp", nccmp_ver)) -esmf_ver=os.getenv("esmf_ver") or "8.6.0" +esmf_ver=os.getenv("esmf_ver") or "8.8.0" load(pathJoin("esmf", esmf_ver)) -nco_ver=os.getenv("nco_ver") or "5.0.6" +nco_ver=os.getenv("nco_ver") or "5.2.4" load(pathJoin("nco", nco_ver)) +setenv("I_MPI_CC", "icx") +setenv("I_MPI_F90", "ifort") + +setenv("CC", "mpiicx") +setenv("FC", "mpiifort") + whatis("Description: UFS_UTILS build environment") diff --git a/modulefiles/build.hercules.intelllvm.lua b/modulefiles/build.hercules.intelllvm.lua index eeddf7966..6e17c86d1 100644 --- a/modulefiles/build.hercules.intelllvm.lua +++ b/modulefiles/build.hercules.intelllvm.lua @@ -2,15 +2,15 @@ help([[ Load environment to compile UFS_UTILS on Hercules using IntelLLVM ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") +prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") -stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" +stack_intel_ver=os.getenv("stack_intel_ver") or "2024.2.1" load(pathJoin("stack-intel", stack_intel_ver)) -stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0" +stack_impi_ver=os.getenv("stack_impi_ver") or "2021.13" load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) -cmake_ver=os.getenv("cmake_ver") or "3.23.1" +cmake_ver=os.getenv("cmake_ver") or "3.27.9" load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" @@ -19,7 +19,7 @@ load(pathJoin("bacio", bacio_ver)) g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) -ip_ver=os.getenv("ip_ver") or "4.3.0" +ip_ver=os.getenv("ip_ver") or "5.1.0" load(pathJoin("ip", ip_ver)) nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" @@ -32,10 +32,10 @@ w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" load(pathJoin("w3emc", w3emc_ver)) -- Uncomment when CHGRES_ALL is ON ---sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" +--sfcio_ver=os.getenv("sfcio_ver") or "1.4.2" --load(pathJoin("sfcio", sfcio_ver)) -sigio_ver=os.getenv("sigio_ver") or "2.3.2" +sigio_ver=os.getenv("sigio_ver") or "2.3.3" load(pathJoin("sigio", sigio_ver)) zlib_ver=os.getenv("zlib_ver") or "1.2.13" @@ -53,16 +53,16 @@ load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" load(pathJoin("nccmp", nccmp_ver)) -esmf_ver=os.getenv("esmf_ver") or "8.6.0" +esmf_ver=os.getenv("esmf_ver") or "8.8.0" load(pathJoin("esmf", esmf_ver)) -nco_ver=os.getenv("nco_ver") or "5.0.6" +nco_ver=os.getenv("nco_ver") or "5.2.4" load(pathJoin("nco", nco_ver)) setenv("I_MPI_CC", "icx") setenv("I_MPI_F90", "ifx") -setenv("CC", "mpiicc") -setenv("FC", "mpiifort") +setenv("CC", "mpiicx") +setenv("FC", "mpiifx") whatis("Description: UFS_UTILS build environment") diff --git a/modulefiles/build.orion.intel.lua b/modulefiles/build.orion.intel.lua index b7fc2befc..d912a67f3 100644 --- a/modulefiles/build.orion.intel.lua +++ b/modulefiles/build.orion.intel.lua @@ -2,15 +2,15 @@ help([[ Load environment to compile UFS_UTILS on Orion using Intel classic ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") +prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") -stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" +stack_intel_ver=os.getenv("stack_intel_ver") or "2024.2.1" load(pathJoin("stack-intel", stack_intel_ver)) -stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0" +stack_impi_ver=os.getenv("stack_impi_ver") or "2021.13" load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) -cmake_ver=os.getenv("cmake_ver") or "3.23.1" +cmake_ver=os.getenv("cmake_ver") or "3.27.9" load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" @@ -19,7 +19,7 @@ load(pathJoin("bacio", bacio_ver)) g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) -ip_ver=os.getenv("ip_ver") or "4.3.0" +ip_ver=os.getenv("ip_ver") or "5.1.0" load(pathJoin("ip", ip_ver)) nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" @@ -32,10 +32,10 @@ w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" load(pathJoin("w3emc", w3emc_ver)) -- Uncomment when CHGRES_ALL is ON ---sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" +--sfcio_ver=os.getenv("sfcio_ver") or "1.4.2" --load(pathJoin("sfcio", sfcio_ver)) -sigio_ver=os.getenv("sigio_ver") or "2.3.2" +sigio_ver=os.getenv("sigio_ver") or "2.3.3" load(pathJoin("sigio", sigio_ver)) zlib_ver=os.getenv("zlib_ver") or "1.2.13" @@ -53,10 +53,16 @@ load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" load(pathJoin("nccmp", nccmp_ver)) -esmf_ver=os.getenv("esmf_ver") or "8.6.0" +esmf_ver=os.getenv("esmf_ver") or "8.8.0" load(pathJoin("esmf", esmf_ver)) -nco_ver=os.getenv("nco_ver") or "5.0.6" +nco_ver=os.getenv("nco_ver") or "5.2.4" load(pathJoin("nco", nco_ver)) +setenv("I_MPI_CC", "icx") +setenv("I_MPI_F90", "ifort") + +setenv("CC", "mpiicx") +setenv("FC", "mpiifort") + whatis("Description: UFS_UTILS build environment") diff --git a/modulefiles/build.orion.intelllvm.lua b/modulefiles/build.orion.intelllvm.lua index e32178289..a2d1cc489 100644 --- a/modulefiles/build.orion.intelllvm.lua +++ b/modulefiles/build.orion.intelllvm.lua @@ -2,15 +2,15 @@ help([[ Load environment to compile UFS_UTILS on Orion using IntelLLVM ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") +prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") -stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" +stack_intel_ver=os.getenv("stack_intel_ver") or "2024.2.1" load(pathJoin("stack-intel", stack_intel_ver)) -stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0" +stack_impi_ver=os.getenv("stack_impi_ver") or "2021.13" load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) -cmake_ver=os.getenv("cmake_ver") or "3.23.1" +cmake_ver=os.getenv("cmake_ver") or "3.27.9" load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" @@ -19,7 +19,7 @@ load(pathJoin("bacio", bacio_ver)) g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) -ip_ver=os.getenv("ip_ver") or "4.3.0" +ip_ver=os.getenv("ip_ver") or "5.1.0" load(pathJoin("ip", ip_ver)) nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" @@ -32,10 +32,10 @@ w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" load(pathJoin("w3emc", w3emc_ver)) -- Uncomment when CHGRES_ALL is ON ---sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" +--sfcio_ver=os.getenv("sfcio_ver") or "1.4.2" --load(pathJoin("sfcio", sfcio_ver)) -sigio_ver=os.getenv("sigio_ver") or "2.3.2" +sigio_ver=os.getenv("sigio_ver") or "2.3.3" load(pathJoin("sigio", sigio_ver)) zlib_ver=os.getenv("zlib_ver") or "1.2.13" @@ -53,16 +53,16 @@ load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" load(pathJoin("nccmp", nccmp_ver)) -esmf_ver=os.getenv("esmf_ver") or "8.6.0" +esmf_ver=os.getenv("esmf_ver") or "8.8.0" load(pathJoin("esmf", esmf_ver)) -nco_ver=os.getenv("nco_ver") or "5.0.6" +nco_ver=os.getenv("nco_ver") or "5.2.4" load(pathJoin("nco", nco_ver)) setenv("I_MPI_CC", "icx") setenv("I_MPI_F90", "ifx") -setenv("CC", "mpiicc") -setenv("FC", "mpiifort") +setenv("CC", "mpiicx") +setenv("FC", "mpiifx") whatis("Description: UFS_UTILS build environment") From 5f62f10ff6133ebcd59ee88c79fdbe7a22ee6057 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 20 Jun 2025 13:44:46 -0500 Subject: [PATCH 37/54] Correct oneapi stack names on MSU --- modulefiles/build.hercules.intel.lua | 4 ++-- modulefiles/build.hercules.intelllvm.lua | 4 ++-- modulefiles/build.orion.intel.lua | 4 ++-- modulefiles/build.orion.intelllvm.lua | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modulefiles/build.hercules.intel.lua b/modulefiles/build.hercules.intel.lua index 7e5d777b2..8a74be8c3 100644 --- a/modulefiles/build.hercules.intel.lua +++ b/modulefiles/build.hercules.intel.lua @@ -4,8 +4,8 @@ Load environment to compile UFS_UTILS on Hercules using Intel Classic prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") -stack_intel_ver=os.getenv("stack_intel_ver") or "2024.2.1" -load(pathJoin("stack-intel", stack_intel_ver)) +stack_oneapi_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" +load(pathJoin("stack-oneapi", stack_oneapi_ver)) stack_impi_ver=os.getenv("stack_impi_ver") or "2021.13" load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) diff --git a/modulefiles/build.hercules.intelllvm.lua b/modulefiles/build.hercules.intelllvm.lua index 6e17c86d1..f757d7f7f 100644 --- a/modulefiles/build.hercules.intelllvm.lua +++ b/modulefiles/build.hercules.intelllvm.lua @@ -4,8 +4,8 @@ Load environment to compile UFS_UTILS on Hercules using IntelLLVM prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") -stack_intel_ver=os.getenv("stack_intel_ver") or "2024.2.1" -load(pathJoin("stack-intel", stack_intel_ver)) +stack_oneapi_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" +load(pathJoin("stack_oneapi", stack_oneapi_ver)) stack_impi_ver=os.getenv("stack_impi_ver") or "2021.13" load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) diff --git a/modulefiles/build.orion.intel.lua b/modulefiles/build.orion.intel.lua index d912a67f3..07e84c257 100644 --- a/modulefiles/build.orion.intel.lua +++ b/modulefiles/build.orion.intel.lua @@ -4,8 +4,8 @@ Load environment to compile UFS_UTILS on Orion using Intel classic prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") -stack_intel_ver=os.getenv("stack_intel_ver") or "2024.2.1" -load(pathJoin("stack-intel", stack_intel_ver)) +stack_oneapi_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" +load(pathJoin("stack-oneapi", stack_oneapi_ver)) stack_impi_ver=os.getenv("stack_impi_ver") or "2021.13" load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) diff --git a/modulefiles/build.orion.intelllvm.lua b/modulefiles/build.orion.intelllvm.lua index a2d1cc489..cd94e58e1 100644 --- a/modulefiles/build.orion.intelllvm.lua +++ b/modulefiles/build.orion.intelllvm.lua @@ -4,8 +4,8 @@ Load environment to compile UFS_UTILS on Orion using IntelLLVM prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") -stack_intel_ver=os.getenv("stack_intel_ver") or "2024.2.1" -load(pathJoin("stack-intel", stack_intel_ver)) +stack_oneapi_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" +load(pathJoin("stack-oneapi", stack_oneapi_ver)) stack_impi_ver=os.getenv("stack_impi_ver") or "2021.13" load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) From bfb726a447bd58d9066310debb927914bbb60efe Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 20 Jun 2025 13:55:27 -0500 Subject: [PATCH 38/54] Fix hercules stack module name --- modulefiles/build.hercules.intelllvm.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/build.hercules.intelllvm.lua b/modulefiles/build.hercules.intelllvm.lua index f757d7f7f..cc340bd31 100644 --- a/modulefiles/build.hercules.intelllvm.lua +++ b/modulefiles/build.hercules.intelllvm.lua @@ -5,7 +5,7 @@ Load environment to compile UFS_UTILS on Hercules using IntelLLVM prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") stack_oneapi_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" -load(pathJoin("stack_oneapi", stack_oneapi_ver)) +load(pathJoin("stack-oneapi", stack_oneapi_ver)) stack_impi_ver=os.getenv("stack_impi_ver") or "2021.13" load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) From c254bd886c2ac96daaed02565a11c2a1a953dcce Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Mon, 23 Jun 2025 12:04:51 +0000 Subject: [PATCH 39/54] Update to spack stack v1.9.2 Fixes #1068 --- modulefiles/build.ursa.gnu.lua | 6 +++--- modulefiles/build.ursa.intelllvm.lua | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modulefiles/build.ursa.gnu.lua b/modulefiles/build.ursa.gnu.lua index 05fd0ab20..72e20f930 100644 --- a/modulefiles/build.ursa.gnu.lua +++ b/modulefiles/build.ursa.gnu.lua @@ -2,7 +2,7 @@ help([[ Load environment to compile UFS_UTILS on Ursa using Gnu ]]) -prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.9.1/envs/ue-gcc-12.4.0/install/modulefiles/Core") +prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.9.2/envs/ue-gcc-12.4.0/install/modulefiles/Core") stack_gcc_ver=os.getenv("stack_gcc_ver") or "12.4.0" load(pathJoin("stack-gcc", stack_gcc_ver)) @@ -25,8 +25,8 @@ load(pathJoin("ip", ip_ver)) nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" load(pathJoin("nemsio", nemsio_ver)) ---sp_ver=os.getenv("sp_ver") or "2.5.0" ---load(pathJoin("sp", sp_ver)) +sp_ver=os.getenv("sp_ver") or "2.5.0" +load(pathJoin("sp", sp_ver)) w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" load(pathJoin("w3emc", w3emc_ver)) diff --git a/modulefiles/build.ursa.intelllvm.lua b/modulefiles/build.ursa.intelllvm.lua index 618b1d18a..c1924b57f 100644 --- a/modulefiles/build.ursa.intelllvm.lua +++ b/modulefiles/build.ursa.intelllvm.lua @@ -2,7 +2,7 @@ help([[ Load environment to compile UFS_UTILS on Ursa using Intelllvm ]]) -prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.2.1/install/modulefiles/Core") +prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.9.2/envs/ue-oneapi-2024.2.1/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" load(pathJoin("stack-oneapi", stack_oneapi_ver)) From 5e9ef19ee49ea61416169b7f26414fdcd028a7a7 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 3 Jul 2025 11:02:27 -0400 Subject: [PATCH 40/54] Update ci versions --- ci/spack.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ci/spack.yaml b/ci/spack.yaml index 567d894a9..784b4e70c 100644 --- a/ci/spack.yaml +++ b/ci/spack.yaml @@ -7,15 +7,14 @@ spack: - netcdf-c@4.9.2 - netcdf-fortran@4.6.1 - bacio@2.4.1 - - g2@3.4.5 - - ip@4.4.0 precision=d + - g2@3.5.1 + - ip@5.1.0 precision=d - nemsio@2.5.4 - - sp@2.5.0 - w3emc@2.10.0 - sigio@2.3.3 - nccmp@1.9.0.1 - parallelio@2.5.10+fortran~pnetcdf - - esmf@8.6.0 + - esmf@8.8.0 view: true concretizer: unify: when_possible From ace1002c300933423c3430c089fd0a48e8876d88 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 3 Jul 2025 11:57:00 -0400 Subject: [PATCH 41/54] Add sp back in to spack.yaml --- ci/spack.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/spack.yaml b/ci/spack.yaml index 784b4e70c..3985e540f 100644 --- a/ci/spack.yaml +++ b/ci/spack.yaml @@ -9,6 +9,7 @@ spack: - bacio@2.4.1 - g2@3.5.1 - ip@5.1.0 precision=d + - sp@2.5.0 - nemsio@2.5.4 - w3emc@2.10.0 - sigio@2.3.3 From 27130cb6219f06bb74be455ac9a7b74a9e57f83c Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 3 Jul 2025 13:45:00 -0400 Subject: [PATCH 42/54] Update to spack-stack 1.9.2 --- modulefiles/build.gaeac6.intel.lua | 2 +- modulefiles/build.hercules.intel.lua | 2 +- modulefiles/build.hercules.intelllvm.lua | 2 +- modulefiles/build.orion.intel.lua | 2 +- modulefiles/build.orion.intelllvm.lua | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modulefiles/build.gaeac6.intel.lua b/modulefiles/build.gaeac6.intel.lua index 2955c59e8..67539ba56 100644 --- a/modulefiles/build.gaeac6.intel.lua +++ b/modulefiles/build.gaeac6.intel.lua @@ -5,7 +5,7 @@ Load environment to compile UFS_UTILS on Gaea C6 using Intel prepend_path("MODULEPATH", "/usw/hpss/modulefiles") load("hsi") -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.9.1/envs/ue-intel-2023.2.0/install/modulefiles/Core") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.9.2/envs/ue-intel-2023.2.0/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" load(pathJoin("stack-intel", stack_intel_ver)) diff --git a/modulefiles/build.hercules.intel.lua b/modulefiles/build.hercules.intel.lua index 8a74be8c3..df0e3d417 100644 --- a/modulefiles/build.hercules.intel.lua +++ b/modulefiles/build.hercules.intel.lua @@ -2,7 +2,7 @@ help([[ Load environment to compile UFS_UTILS on Hercules using Intel Classic ]]) -prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") +prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.2/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") stack_oneapi_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" load(pathJoin("stack-oneapi", stack_oneapi_ver)) diff --git a/modulefiles/build.hercules.intelllvm.lua b/modulefiles/build.hercules.intelllvm.lua index cc340bd31..78b64d13f 100644 --- a/modulefiles/build.hercules.intelllvm.lua +++ b/modulefiles/build.hercules.intelllvm.lua @@ -2,7 +2,7 @@ help([[ Load environment to compile UFS_UTILS on Hercules using IntelLLVM ]]) -prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") +prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.2/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") stack_oneapi_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" load(pathJoin("stack-oneapi", stack_oneapi_ver)) diff --git a/modulefiles/build.orion.intel.lua b/modulefiles/build.orion.intel.lua index 07e84c257..339b4385f 100644 --- a/modulefiles/build.orion.intel.lua +++ b/modulefiles/build.orion.intel.lua @@ -2,7 +2,7 @@ help([[ Load environment to compile UFS_UTILS on Orion using Intel classic ]]) -prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") +prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.2/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") stack_oneapi_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" load(pathJoin("stack-oneapi", stack_oneapi_ver)) diff --git a/modulefiles/build.orion.intelllvm.lua b/modulefiles/build.orion.intelllvm.lua index cd94e58e1..c78e8820c 100644 --- a/modulefiles/build.orion.intelllvm.lua +++ b/modulefiles/build.orion.intelllvm.lua @@ -2,7 +2,7 @@ help([[ Load environment to compile UFS_UTILS on Orion using IntelLLVM ]]) -prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") +prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.2/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") stack_oneapi_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" load(pathJoin("stack-oneapi", stack_oneapi_ver)) From 3c6005ef19563a12566104e7ebbfce8ee682da18 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 3 Jul 2025 14:03:00 -0400 Subject: [PATCH 43/54] Update esmf suite to -D (v8.8.0) --- modulefiles/build.wcoss2.intel.lua | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/modulefiles/build.wcoss2.intel.lua b/modulefiles/build.wcoss2.intel.lua index e5f179e73..4109c6f23 100644 --- a/modulefiles/build.wcoss2.intel.lua +++ b/modulefiles/build.wcoss2.intel.lua @@ -27,10 +27,13 @@ libpng_ver=os.getenv("libpng_ver") or "1.6.37" load(pathJoin("libpng", libpng_ver)) hdf5_ver=os.getenv("hdf5_ver") or "1.14.0" -load(pathJoin("hdf5-C", hdf5_ver)) +load(pathJoin("hdf5-D", hdf5_ver)) + +pnetcdf_ver=os.getenv("pnetcdf_ver") or "1.12.2" +load(pathJoin("pnetcdf-D", pnetcdf_ver)) netcdf_ver=os.getenv("netcdf_ver") or "4.9.2" -load(pathJoin("netcdf-C", netcdf_ver)) +load(pathJoin("netcdf-D", netcdf_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) @@ -72,7 +75,7 @@ load(pathJoin("gsl", gsl_ver)) nco_ver=os.getenv("nco_ver") or "5.0.6" load(pathJoin("nco", nco_ver)) -esmf_ver=os.getenv("esmf_ver") or "8.6.0" -load(pathJoin("esmf-C", esmf_ver)) +esmf_ver=os.getenv("esmf_ver") or "8.8.0" +load(pathJoin("esmf-D", esmf_ver)) whatis("Description: UFS_UTILS build environment") From 0c11c7d6610cfdac8ceec2e09a811d9f2a059225 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 3 Jul 2025 21:02:34 +0000 Subject: [PATCH 44/54] Add minimal hera support back in --- modulefiles/build.hera.intel.lua | 62 ++++++++++++++++++++++++++++++++ sorc/machine-setup.sh | 12 ++++--- 2 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 modulefiles/build.hera.intel.lua diff --git a/modulefiles/build.hera.intel.lua b/modulefiles/build.hera.intel.lua new file mode 100644 index 000000000..2fadb07dd --- /dev/null +++ b/modulefiles/build.hera.intel.lua @@ -0,0 +1,62 @@ +help([[ +Load environment to compile UFS_UTILS on Ursa using Intelllvm +]]) + +prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.9.2/envs/ue-oneapi-2024.2.1/install/modulefiles/Core") + +stack_intel_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" +load(pathJoin("stack-oneapi", stack_oneapi_ver)) + +stack_oneapi_mpi_ver=os.getenv("stack_oneapi_mpi_ver") or "2021.13" +load(pathJoin("stack-intel-oneapi-mpi", stack_oneapi_mpi_ver)) + +cmake_ver=os.getenv("cmake_ver") or "3.27.9" +load(pathJoin("cmake", cmake_ver)) + +bacio_ver=os.getenv("bacio_ver") or "2.4.1" +load(pathJoin("bacio", bacio_ver)) + +g2_ver=os.getenv("g2_ver") or "3.5.1" +load(pathJoin("g2", g2_ver)) + +ip_ver=os.getenv("ip_ver") or "5.1.0" +load(pathJoin("ip", ip_ver)) + +nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" +load(pathJoin("nemsio", nemsio_ver)) + +sp_ver=os.getenv("sp_ver") or "2.5.0" +load(pathJoin("sp", sp_ver)) + +w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" +load(pathJoin("w3emc", w3emc_ver)) + +-- Uncomment when CHGRES_ALL is ON +--sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" +--load(pathJoin("sfcio", sfcio_ver)) + +sigio_ver=os.getenv("sigio_ver") or "2.3.3" +load(pathJoin("sigio", sigio_ver)) + +netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" +load(pathJoin("netcdf-c", netcdf_c_ver)) + +netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1" +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) + +nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" +load(pathJoin("nccmp", nccmp_ver)) + +esmf_ver=os.getenv("esmf_ver") or "8.6.1" +load(pathJoin("esmf", esmf_ver)) + +nco_ver=os.getenv("nco_ver") or "5.2.4" +load(pathJoin("nco", nco_ver)) + +setenv("I_MPI_CC", "icx") +setenv("I_MPI_FC", "ifort") + +setenv("CC", "mpiicc") +setenv("FC", "mpiifort") + +whatis("Description: UFS_UTILS build environment") diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index 067ef0114..46f712b49 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -35,14 +35,18 @@ elif [[ -d /opt/spack-stack && -v SINGULARITY_CONTAINER ]]; then source /usr/lmod/lmod/init/$__ms_shell target=container module purge -elif [[ -d /scratch3 ]] ; then - # We are on NOAA Ursa +elif [[ -d /scratch3 ]]; then + # We are on NOAA Hera or Ursa + mount=$(findmnt -n -o SOURCE /home) + if [[ ${mount} =~ "ursa" ]]; then + target=ursa + else + target=hera + fi if ( ! eval module help > /dev/null 2>&1 ) ; then echo load the module command 1>&2 source /apps/lmod/lmod/init/$__ms_shell fi - target=ursa - module purge elif [[ "$(hostname)" == "gaea6"* || "$(hostname)" =~ c6n[0-9]+ ]] && [[ -d /gpfs/f6 ]] ; then target=gaeac6 source /opt/cray/pe/lmod/8.7.31/init/$__ms_shell From d0cc2b4a15d9cef1a059095058223c72207454e6 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 8 Jul 2025 13:37:48 +0000 Subject: [PATCH 45/54] Revert ESMF to 8.6.0 on WCOSS2 --- modulefiles/build.wcoss2.intel.lua | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/modulefiles/build.wcoss2.intel.lua b/modulefiles/build.wcoss2.intel.lua index 4109c6f23..e5f179e73 100644 --- a/modulefiles/build.wcoss2.intel.lua +++ b/modulefiles/build.wcoss2.intel.lua @@ -27,13 +27,10 @@ libpng_ver=os.getenv("libpng_ver") or "1.6.37" load(pathJoin("libpng", libpng_ver)) hdf5_ver=os.getenv("hdf5_ver") or "1.14.0" -load(pathJoin("hdf5-D", hdf5_ver)) - -pnetcdf_ver=os.getenv("pnetcdf_ver") or "1.12.2" -load(pathJoin("pnetcdf-D", pnetcdf_ver)) +load(pathJoin("hdf5-C", hdf5_ver)) netcdf_ver=os.getenv("netcdf_ver") or "4.9.2" -load(pathJoin("netcdf-D", netcdf_ver)) +load(pathJoin("netcdf-C", netcdf_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) @@ -75,7 +72,7 @@ load(pathJoin("gsl", gsl_ver)) nco_ver=os.getenv("nco_ver") or "5.0.6" load(pathJoin("nco", nco_ver)) -esmf_ver=os.getenv("esmf_ver") or "8.8.0" -load(pathJoin("esmf-D", esmf_ver)) +esmf_ver=os.getenv("esmf_ver") or "8.6.0" +load(pathJoin("esmf-C", esmf_ver)) whatis("Description: UFS_UTILS build environment") From 4a1fb1e65739a47232ea6fa1e0a5cafdd49e131a Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 8 Jul 2025 15:52:30 +0000 Subject: [PATCH 46/54] Upgrade ip to 5.2.0 on WCOSS2 --- modulefiles/build.wcoss2.intel.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/build.wcoss2.intel.lua b/modulefiles/build.wcoss2.intel.lua index e5f179e73..be8d25fc5 100644 --- a/modulefiles/build.wcoss2.intel.lua +++ b/modulefiles/build.wcoss2.intel.lua @@ -51,7 +51,7 @@ load(pathJoin("sigio", sigio_ver)) sp_ver=os.getenv("sp_ver") or "2.4.0" load(pathJoin("sp", sp_ver)) -ip_ver=os.getenv("ip_ver") or "4.0.0" +ip_ver=os.getenv("ip_ver") or "5.2.0" load(pathJoin("ip", ip_ver)) g2_ver=os.getenv("g2_ver") or "3.5.1" From cb0d6486c839012422b45938756d6bafac317893 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 8 Jul 2025 15:59:52 +0000 Subject: [PATCH 47/54] Revert ESMF to 8.6.0 in ci/spack.yaml --- ci/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/spack.yaml b/ci/spack.yaml index 3985e540f..fffd14adb 100644 --- a/ci/spack.yaml +++ b/ci/spack.yaml @@ -15,7 +15,7 @@ spack: - sigio@2.3.3 - nccmp@1.9.0.1 - parallelio@2.5.10+fortran~pnetcdf - - esmf@8.8.0 + - esmf@8.6.0 view: true concretizer: unify: when_possible From ac94fa767f761075bbcd4bf84073aa49ee9930a5 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 9 Jul 2025 12:58:08 +0000 Subject: [PATCH 48/54] Upgrade to ESMF 8.8.0 on WCOSS2 --- modulefiles/build.wcoss2.intel.lua | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/modulefiles/build.wcoss2.intel.lua b/modulefiles/build.wcoss2.intel.lua index e5f179e73..dc56ea127 100644 --- a/modulefiles/build.wcoss2.intel.lua +++ b/modulefiles/build.wcoss2.intel.lua @@ -14,9 +14,13 @@ load(pathJoin("craype", craype_ver)) intel_ver=os.getenv("intel_ver") or "19.1.3.304" load(pathJoin("intel", intel_ver)) -cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.12" +cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.19" load(pathJoin("cray-mpich", cray_mpich_ver)) +-- C MPI bindings needed by netCDF are not included by default (used by fregrid, make_solo_mosaic, and make_hgrid). +-- TODO when this is fixed in the cray "cc" wrappers, remove this +setenv("CMAKE_FLAGS", "-DCMAKE_C_FLAGS=/opt/cray/pe/mpich/8.1.19/ofi/intel/19.0/lib/libmpifort_intel.so /opt/cray/pe/mpich/8.1.19/ofi/intel/19.0/lib/libmpi_intel.so ") + libjpeg_ver=os.getenv("libjpeg_ver") or "9c" load(pathJoin("libjpeg", libjpeg_ver)) @@ -27,10 +31,13 @@ libpng_ver=os.getenv("libpng_ver") or "1.6.37" load(pathJoin("libpng", libpng_ver)) hdf5_ver=os.getenv("hdf5_ver") or "1.14.0" -load(pathJoin("hdf5-C", hdf5_ver)) +load(pathJoin("hdf5-D", hdf5_ver)) netcdf_ver=os.getenv("netcdf_ver") or "4.9.2" -load(pathJoin("netcdf-C", netcdf_ver)) +load(pathJoin("netcdf-D", netcdf_ver)) + +netcdf_ver=os.getenv("pnetcdf_ver") or "1.12.2" +load(pathJoin("pnetcdf-D", netcdf_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) @@ -51,7 +58,7 @@ load(pathJoin("sigio", sigio_ver)) sp_ver=os.getenv("sp_ver") or "2.4.0" load(pathJoin("sp", sp_ver)) -ip_ver=os.getenv("ip_ver") or "4.0.0" +ip_ver=os.getenv("ip_ver") or "5.2.0" load(pathJoin("ip", ip_ver)) g2_ver=os.getenv("g2_ver") or "3.5.1" @@ -72,7 +79,7 @@ load(pathJoin("gsl", gsl_ver)) nco_ver=os.getenv("nco_ver") or "5.0.6" load(pathJoin("nco", nco_ver)) -esmf_ver=os.getenv("esmf_ver") or "8.6.0" -load(pathJoin("esmf-C", esmf_ver)) +esmf_ver=os.getenv("esmf_ver") or "8.8.0" +load(pathJoin("esmf-D", esmf_ver)) whatis("Description: UFS_UTILS build environment") From 29982b7836dd43b22fff0131e672c84c9b5c812f Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 9 Jul 2025 12:59:50 +0000 Subject: [PATCH 49/54] Re-upgrade ci builds to 8.8.0 --- ci/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/spack.yaml b/ci/spack.yaml index fffd14adb..3985e540f 100644 --- a/ci/spack.yaml +++ b/ci/spack.yaml @@ -15,7 +15,7 @@ spack: - sigio@2.3.3 - nccmp@1.9.0.1 - parallelio@2.5.10+fortran~pnetcdf - - esmf@8.6.0 + - esmf@8.8.0 view: true concretizer: unify: when_possible From f1edc5662b00204c5069611026aa1bef9fa96061 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 9 Jul 2025 15:32:41 +0000 Subject: [PATCH 50/54] Replaced CMAKE option with an LD_LIBRARY_PATH update; recommended by Steve Bongiovanni --- modulefiles/build.wcoss2.intel.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modulefiles/build.wcoss2.intel.lua b/modulefiles/build.wcoss2.intel.lua index dc56ea127..01f5e92bd 100644 --- a/modulefiles/build.wcoss2.intel.lua +++ b/modulefiles/build.wcoss2.intel.lua @@ -17,9 +17,9 @@ load(pathJoin("intel", intel_ver)) cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.19" load(pathJoin("cray-mpich", cray_mpich_ver)) --- C MPI bindings needed by netCDF are not included by default (used by fregrid, make_solo_mosaic, and make_hgrid). --- TODO when this is fixed in the cray "cc" wrappers, remove this -setenv("CMAKE_FLAGS", "-DCMAKE_C_FLAGS=/opt/cray/pe/mpich/8.1.19/ofi/intel/19.0/lib/libmpifort_intel.so /opt/cray/pe/mpich/8.1.19/ofi/intel/19.0/lib/libmpi_intel.so ") +-- Need the cray library path for C MPI libraries +local cray_lib_path=os.getenv("CRAY_LD_LIBRARY_PATH") or "" +prepend_path("LD_LIBRARY_PATH", cray_lib_path) libjpeg_ver=os.getenv("libjpeg_ver") or "9c" load(pathJoin("libjpeg", libjpeg_ver)) From 3214857de0377c57cba656532ba6475b7fc70d51 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 16 Jul 2025 13:53:52 +0000 Subject: [PATCH 51/54] Update nccmp module on WCOSS2 to align with the netcdf-D suite --- reg_tests/chgres_cube/driver.wcoss2.sh | 3 +-- reg_tests/cpld_gridgen/rt.sh | 3 +-- reg_tests/global_cycle/driver.wcoss2.sh | 3 +-- reg_tests/grid_gen/driver.wcoss2.sh | 3 +-- reg_tests/ocnice_prep/rt.sh | 3 +-- reg_tests/regrid_sfc/driver.sh | 3 +-- reg_tests/rt.sh | 10 +++++----- reg_tests/weight_gen/driver.wcoss2.sh | 3 +-- 8 files changed, 12 insertions(+), 19 deletions(-) diff --git a/reg_tests/chgres_cube/driver.wcoss2.sh b/reg_tests/chgres_cube/driver.wcoss2.sh index e38af7278..6cfc61640 100755 --- a/reg_tests/chgres_cube/driver.wcoss2.sh +++ b/reg_tests/chgres_cube/driver.wcoss2.sh @@ -30,8 +30,7 @@ source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.$compiler # load nccmp -module load netcdf -module load nccmp +module load nccmp-D/1.9.0.1 module list export OUTDIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" diff --git a/reg_tests/cpld_gridgen/rt.sh b/reg_tests/cpld_gridgen/rt.sh index 41d932efa..ca499347a 100755 --- a/reg_tests/cpld_gridgen/rt.sh +++ b/reg_tests/cpld_gridgen/rt.sh @@ -169,8 +169,7 @@ fi module use $PATHTR/modulefiles module load build.$target.$compiler if [[ $target = wcoss2 ]]; then - module load netcdf - module load nccmp + module load nccmp-D/1.9.0.1 fi set +x module list diff --git a/reg_tests/global_cycle/driver.wcoss2.sh b/reg_tests/global_cycle/driver.wcoss2.sh index 2da826570..606bb5234 100755 --- a/reg_tests/global_cycle/driver.wcoss2.sh +++ b/reg_tests/global_cycle/driver.wcoss2.sh @@ -25,8 +25,7 @@ compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.$compiler -module load netcdf -module load nccmp +module load nccmp-D/1.9.0.1 module list WORK_DIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" diff --git a/reg_tests/grid_gen/driver.wcoss2.sh b/reg_tests/grid_gen/driver.wcoss2.sh index 013ae3648..50adf01c1 100755 --- a/reg_tests/grid_gen/driver.wcoss2.sh +++ b/reg_tests/grid_gen/driver.wcoss2.sh @@ -24,8 +24,7 @@ compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.$compiler -module load netcdf -module load nccmp +module load nccmp-D/1.9.0.1 module list set -x diff --git a/reg_tests/ocnice_prep/rt.sh b/reg_tests/ocnice_prep/rt.sh index 8907ac424..b2f04d28c 100755 --- a/reg_tests/ocnice_prep/rt.sh +++ b/reg_tests/ocnice_prep/rt.sh @@ -176,8 +176,7 @@ fi module use $PATHTR/modulefiles module load build.$target.$compiler if [[ $target = wcoss2 ]]; then - module load netcdf - module load nccmp/1.8.9.0 + module load nccmp-D/1.9.0.1 fi set +x module list diff --git a/reg_tests/regrid_sfc/driver.sh b/reg_tests/regrid_sfc/driver.sh index babc44dfa..0505bdf89 100755 --- a/reg_tests/regrid_sfc/driver.sh +++ b/reg_tests/regrid_sfc/driver.sh @@ -33,8 +33,7 @@ if [[ "$compiler" == "intelllvm" ]]; then fi module load build.$target.$compiler if [[ "$target" == "wcoss2" ]];then - module load netcdf - module load nccmp + module load nccmp-D/1.9.0.1 fi set +x module list diff --git a/reg_tests/rt.sh b/reg_tests/rt.sh index 0d008a60d..b4b609c23 100755 --- a/reg_tests/rt.sh +++ b/reg_tests/rt.sh @@ -15,14 +15,14 @@ wait_for_fin() { ulimit -s unlimited -export MAILTO= +export MAILTO=david.huber@noaa.gov # Directory to download UFS_UTILS to and run the consistency tests -export WORK_DIR= +export WORK_DIR=/lfs/h2/emc/stmp/david.huber/ufs_utils_rt -export PROJECT_CODE= -export QUEUE= -TIMEOUT_LIMIT=3600 +export PROJECT_CODE=GFS-DEV +export QUEUE=dev +TIMEOUT_LIMIT=7200 mkdir -p ${WORK_DIR} cd ${WORK_DIR} diff --git a/reg_tests/weight_gen/driver.wcoss2.sh b/reg_tests/weight_gen/driver.wcoss2.sh index 720100d04..a6536d779 100755 --- a/reg_tests/weight_gen/driver.wcoss2.sh +++ b/reg_tests/weight_gen/driver.wcoss2.sh @@ -35,8 +35,7 @@ compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.$compiler -module load netcdf -module load nccmp +module load nccmp-D/1.9.0.1 module list export DATA="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" From a4fb2b3b655e1eee1c1e508e3075258d00dadb1d Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 16 Jul 2025 13:59:33 +0000 Subject: [PATCH 52/54] Revert erroneous update to reg_tests/rt.sh --- reg_tests/rt.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/reg_tests/rt.sh b/reg_tests/rt.sh index b4b609c23..0d008a60d 100755 --- a/reg_tests/rt.sh +++ b/reg_tests/rt.sh @@ -15,14 +15,14 @@ wait_for_fin() { ulimit -s unlimited -export MAILTO=david.huber@noaa.gov +export MAILTO= # Directory to download UFS_UTILS to and run the consistency tests -export WORK_DIR=/lfs/h2/emc/stmp/david.huber/ufs_utils_rt +export WORK_DIR= -export PROJECT_CODE=GFS-DEV -export QUEUE=dev -TIMEOUT_LIMIT=7200 +export PROJECT_CODE= +export QUEUE= +TIMEOUT_LIMIT=3600 mkdir -p ${WORK_DIR} cd ${WORK_DIR} From e376089a650851af95adf68e5525dc1f5cfb5f57 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 18 Jul 2025 15:54:44 -0500 Subject: [PATCH 53/54] Update the grib-util and wgrib2 modules for Orion --- reg_tests/snow2mdl/driver.orion.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reg_tests/snow2mdl/driver.orion.sh b/reg_tests/snow2mdl/driver.orion.sh index 86735d6bb..df6415c22 100755 --- a/reg_tests/snow2mdl/driver.orion.sh +++ b/reg_tests/snow2mdl/driver.orion.sh @@ -23,8 +23,8 @@ set -x source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.intelllvm -module load grib-util/1.3.0 -module load wgrib2/2.0.8 +module load grib-util/1.4.0 +module load wgrib2/3.6.0 module load prod_util/2.1.1 module list From 985f8bdfd61737abdaa1b62de73df50e909df66d Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 29 Jul 2025 09:19:31 -0500 Subject: [PATCH 54/54] Update grib-utils and wgrib2 module for Orion's ice_blend driver --- reg_tests/ice_blend/driver.orion.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reg_tests/ice_blend/driver.orion.sh b/reg_tests/ice_blend/driver.orion.sh index 8ab6a7fe7..afeaf16cd 100755 --- a/reg_tests/ice_blend/driver.orion.sh +++ b/reg_tests/ice_blend/driver.orion.sh @@ -32,8 +32,8 @@ set -x source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.intelllvm -module load grib-util/1.3.0 -module load wgrib2/2.0.8 +module load grib-util/1.4.0 +module load wgrib2/3.6.0 module list ulimit -s unlimited