From 3c8f2163b3acc59ecf5ed06cc08a5cb79f20cc75 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Sat, 8 Oct 2022 07:08:17 +0000 Subject: [PATCH 1/5] Remove existing aerosol output Due to a change, GOCART currently will not clobber existing output files, resulting in a model crash when attempting to rerun. So, we temporarily remove the aerosol output until an option is introduced to allow clobber. --- ush/forecast_postdet.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 7a2ba0e7adb..4b9e511b720 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -1014,6 +1014,13 @@ GOCART_postdet() { HH=$(echo $VDATE | cut -c9-10) SS=$((10#$HH*3600)) + # + # Temporarily delete existing files due to noclobber in GOCART + # + if [[ -e $COMOUTaero/gocart.inst_aod.${YYYY}${MM}${DD}_${HH}00z.nc4 ]]; then + rm $COMOUTaero/gocart.inst_aod.${YYYY}${MM}${DD}_${HH}00z.nc4 + fi + $NLN $COMOUTaero/gocart.inst_aod.${YYYY}${MM}${DD}_${HH}00z.nc4 $DATA/gocart.inst_aod.${YYYY}${MM}${DD}_${HH}00z.nc4 done } From a4a978b2b3d3fbd89c4ac64e9ba97584e268db60 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Sat, 8 Oct 2022 07:16:14 +0000 Subject: [PATCH 2/5] Update UFS to Sept 9 Updates the UFS model to Sept 9. Some modules are updated to match the UFS versions. This update is still not fully tested and additional changes may be necessary. --- Externals.cfg | 2 +- modulefiles/module_base.hera.lua | 6 ++++-- modulefiles/module_base.orion.lua | 6 ++++-- modulefiles/modulefile.ww3.hera.lua | 1 + modulefiles/modulefile.ww3.orion.lua | 1 + sorc/build_ufs.sh | 2 +- sorc/checkout.sh | 7 +++---- ush/parsing_namelists_CICE.sh | 2 -- 8 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index e8851e2a6b9..3bc53ffe64a 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,7 +1,7 @@ # External sub-modules of global-workflow [UFS] -tag = Prototype-P8 +tag = 3c3548d local_path = sorc/ufs_model.fd repo_url = https://github.com/ufs-community/ufs-weather-model.git protocol = git diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua index 14c523c5180..5126fdfd464 100644 --- a/modulefiles/module_base.hera.lua +++ b/modulefiles/module_base.hera.lua @@ -8,6 +8,8 @@ load(pathJoin("hpc", "1.1.0")) load(pathJoin("hpc-intel", "18.0.5.274")) load(pathJoin("hpc-impi", "2018.0.4")) +load(pathJoin("cmake", "3.20.1")) + load(pathJoin("hpss", "hpss")) load(pathJoin("nco", "4.9.1")) load(pathJoin("gempak", "7.4.2")) @@ -25,8 +27,8 @@ load(pathJoin("png", "1.6.35")) load(pathJoin("hdf5", "1.10.6")) load(pathJoin("netcdf", "4.7.4")) -load(pathJoin("pio", "2.5.2")) -load(pathJoin("esmf", "8.2.1b04")) +load(pathJoin("pio", "2.5.7")) +load(pathJoin("esmf", "8.3.0b09")) load(pathJoin("fms", "2021.03")) load(pathJoin("bacio", "2.4.1")) diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua index bdb85d7430e..98fc476d0c4 100644 --- a/modulefiles/module_base.orion.lua +++ b/modulefiles/module_base.orion.lua @@ -8,6 +8,8 @@ load(pathJoin("hpc", "1.1.0")) load(pathJoin("hpc-intel", "2018.4")) load(pathJoin("hpc-impi", "2018.4")) +load(pathJoin("cmake", "3.22.1")) + load(pathJoin("nco", "4.8.1")) load(pathJoin("gempak", "7.5.1")) load(pathJoin("ncl", "6.6.2")) @@ -24,8 +26,8 @@ load(pathJoin("png", "1.6.35")) load(pathJoin("hdf5", "1.10.6")) load(pathJoin("netcdf", "4.7.4")) -load(pathJoin("pio", "2.5.2")) -load(pathJoin("esmf", "8.2.1b04")) +load(pathJoin("pio", "2.5.7")) +load(pathJoin("esmf", "8.3.0b09")) load(pathJoin("fms", "2021.03")) load(pathJoin("bacio", "2.4.1")) diff --git a/modulefiles/modulefile.ww3.hera.lua b/modulefiles/modulefile.ww3.hera.lua index 287f02d0642..0e1730babd1 100644 --- a/modulefiles/modulefile.ww3.hera.lua +++ b/modulefiles/modulefile.ww3.hera.lua @@ -21,3 +21,4 @@ load(pathJoin("hdf5", "1.10.6")) load(pathJoin("netcdf", "4.7.4")) load(pathJoin("w3nco", "2.4.1")) +load(pathJoin("w3emc", "2.9.2")) diff --git a/modulefiles/modulefile.ww3.orion.lua b/modulefiles/modulefile.ww3.orion.lua index 2a1409f9f0a..332572f9e94 100644 --- a/modulefiles/modulefile.ww3.orion.lua +++ b/modulefiles/modulefile.ww3.orion.lua @@ -23,3 +23,4 @@ load(pathJoin("hdf5", "1.10.6")) load(pathJoin("netcdf", "4.7.4")) load(pathJoin("w3nco", "2.4.1")) +load(pathJoin("w3emc", "2.9.2")) diff --git a/sorc/build_ufs.sh b/sorc/build_ufs.sh index 4225debff9b..df1378ca662 100755 --- a/sorc/build_ufs.sh +++ b/sorc/build_ufs.sh @@ -33,6 +33,6 @@ CLEAN_AFTER=NO ./tests/compile.sh $MACHINE_ID "$MAKE_OPT" $COMPILE_NR $CLEAN_BEFORE $CLEAN_AFTER mv ./tests/fv3_${COMPILE_NR}.exe ./tests/ufs_model.x -mv ./tests/modules.fv3_${COMPILE_NR} ./tests/modules.ufs_model +mv ./tests/modules.fv3_${COMPILE_NR}.lua ./tests/modules.ufs_model.lua exit 0 diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 441cd781244..d6326e5da9a 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -107,7 +107,6 @@ checkout_gsi="NO" checkout_gdas="NO" checkout_gtg="NO" checkout_wafs="NO" -ufs_model_hash="Prototype-P8" # Parse command line arguments while getopts ":chgum:o" option; do @@ -153,9 +152,9 @@ mkdir -p "${logdir}" # The checkout version should always be a speciifc commit (hash or tag), not a branch errs=0 checkout "gfs_utils.fd" "https://github.com/NOAA-EMC/gfs-utils" "7bf599f" ; errs=$((errs + $?)) -checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash}"; errs=$((errs + $?)) -checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "8b990c0" ; errs=$((errs + $?)) -checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git" "c267780" ; errs=$((errs + $?)) +checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash:-3c3548d}" ; errs=$((errs + $?)) +checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "8b990c0" ; errs=$((errs + $?)) +checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git" "c267780" ; errs=$((errs + $?)) if [[ ${checkout_gsi} == "YES" ]]; then checkout "gsi_enkf.fd" "https://github.com/NOAA-EMC/GSI.git" "67f5ab4"; errs=$((errs + $?)) diff --git a/ush/parsing_namelists_CICE.sh b/ush/parsing_namelists_CICE.sh index f7e00fd0700..64a711bffc3 100755 --- a/ush/parsing_namelists_CICE.sh +++ b/ush/parsing_namelists_CICE.sh @@ -80,8 +80,6 @@ cat > ice_in < Date: Mon, 17 Oct 2022 10:45:44 +0000 Subject: [PATCH 3/5] Adjust fcst resources on Hera for aerosols Hera has less memory per core than other machines. When UFS is run with GOCART, this results in crashes due to the memory limit being reached. To avoid this, the number of threads is increased on Hera when running with aerosols. --- parm/config/config.resources | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/parm/config/config.resources b/parm/config/config.resources index 21ec6d2b5bd..99f686ba700 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -311,6 +311,11 @@ elif [ ${step} = "fcst" ]; then # Aerosol model only uses the atm model PETS export CHMPETS=${ATMPETS} # Aerosol model runs on same PETs as ATM, so don't add to $NTASKS_TOT + + if [[ ${machine} == "HERA" ]]; then + # Need more memory on Hera for aerosols, so increase threads to spread it out + nth_fv3_gfs=${nth_fv3_gfs:-4} + fi fi # If using in-line post, add the write tasks to the ATMPETS From f963b0aa33ca309fab87242cd36241eac9e95607 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Mon, 17 Oct 2022 11:05:12 +0000 Subject: [PATCH 4/5] Style fixes --- ush/forecast_postdet.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 4b9e511b720..e74fcac105e 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -1017,8 +1017,8 @@ GOCART_postdet() { # # Temporarily delete existing files due to noclobber in GOCART # - if [[ -e $COMOUTaero/gocart.inst_aod.${YYYY}${MM}${DD}_${HH}00z.nc4 ]]; then - rm $COMOUTaero/gocart.inst_aod.${YYYY}${MM}${DD}_${HH}00z.nc4 + if [[ -e "${COMOUTaero}/gocart.inst_aod.${YYYY}${MM}${DD}_${HH}00z.nc4" ]]; then + rm "${COMOUTaero}/gocart.inst_aod.${YYYY}${MM}${DD}_${HH}00z.nc4" fi $NLN $COMOUTaero/gocart.inst_aod.${YYYY}${MM}${DD}_${HH}00z.nc4 $DATA/gocart.inst_aod.${YYYY}${MM}${DD}_${HH}00z.nc4 From 2575eaa74a9aa46eadad454c69d56d806f3626bb Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Mon, 17 Oct 2022 09:33:18 -0500 Subject: [PATCH 5/5] Remove unneeded w3nco module load With the move to a w3emc that includes w3nco, the latter no longer needs to be loaded to build WW3 pre/post. Additionally, added the w3emc module loads for the two machines that were not supported when this update process started. --- modulefiles/modulefile.ww3.hera.lua | 1 - modulefiles/modulefile.ww3.orion.lua | 1 - modulefiles/modulefile.ww3.s4.lua | 2 +- modulefiles/modulefile.ww3.wcoss2.lua | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/modulefiles/modulefile.ww3.hera.lua b/modulefiles/modulefile.ww3.hera.lua index 0e1730babd1..9b1ab7492ed 100644 --- a/modulefiles/modulefile.ww3.hera.lua +++ b/modulefiles/modulefile.ww3.hera.lua @@ -20,5 +20,4 @@ load(pathJoin("g2", "3.4.5")) load(pathJoin("hdf5", "1.10.6")) load(pathJoin("netcdf", "4.7.4")) -load(pathJoin("w3nco", "2.4.1")) load(pathJoin("w3emc", "2.9.2")) diff --git a/modulefiles/modulefile.ww3.orion.lua b/modulefiles/modulefile.ww3.orion.lua index 332572f9e94..de1da6b4f95 100644 --- a/modulefiles/modulefile.ww3.orion.lua +++ b/modulefiles/modulefile.ww3.orion.lua @@ -22,5 +22,4 @@ load(pathJoin("g2", "3.4.5")) load(pathJoin("hdf5", "1.10.6")) load(pathJoin("netcdf", "4.7.4")) -load(pathJoin("w3nco", "2.4.1")) load(pathJoin("w3emc", "2.9.2")) diff --git a/modulefiles/modulefile.ww3.s4.lua b/modulefiles/modulefile.ww3.s4.lua index 06aaf6c3fb0..0a8dcb25ff5 100644 --- a/modulefiles/modulefile.ww3.s4.lua +++ b/modulefiles/modulefile.ww3.s4.lua @@ -18,4 +18,4 @@ load(pathJoin("g2", "3.4.1")) load(pathJoin("hdf5", "1.10.6")) load(pathJoin("netcdf", "4.7.4")) -load(pathJoin("w3nco", "2.4.1")) +load(pathJoin("w3emc", "2.9.2")) diff --git a/modulefiles/modulefile.ww3.wcoss2.lua b/modulefiles/modulefile.ww3.wcoss2.lua index 8923ee6c7ab..3536428d31f 100644 --- a/modulefiles/modulefile.ww3.wcoss2.lua +++ b/modulefiles/modulefile.ww3.wcoss2.lua @@ -19,4 +19,4 @@ load(pathJoin("g2", "3.4.5")) load(pathJoin("hdf5", "1.10.6")) load(pathJoin("netcdf", "4.7.4")) -load(pathJoin("w3nco", "2.4.1")) +load(pathJoin("w3emc", "2.9.2"))