From cd3778fdfa06b73a76e1e1bc026ec86b355718d2 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Thu, 15 Jun 2023 11:46:25 -0400 Subject: [PATCH 01/13] update Gaea modulefile --- modulefiles/build_gaea_intel.lua | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/modulefiles/build_gaea_intel.lua b/modulefiles/build_gaea_intel.lua index 4e5cf881bb..3bc053f315 100644 --- a/modulefiles/build_gaea_intel.lua +++ b/modulefiles/build_gaea_intel.lua @@ -7,12 +7,10 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Gaea ]===]) load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1")) -prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-2021.3.0_noarch/modulefiles/stack") +prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack") load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) -load(pathJoin("intel", os.getenv("intel_ver") or "2021.3.0")) -load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2021.3.0")) -load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.11")) -load(pathJoin("gcc", os.getenv("gcc_ver") or "8.3.0")) +load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2")) +load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20")) load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37")) load("srw_common") From 61cce4b36a0b0c066d638763ca834e04cf32ca37 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Tue, 11 Jul 2023 03:02:38 +0000 Subject: [PATCH 02/13] updated modulefiles and scripts for Gaea c3,c4 --- etc/lmod-setup.csh | 6 +----- modulefiles/build_gaea_intel.lua | 2 +- modulefiles/tasks/gaea/plot_allvars.local.lua | 4 ++++ modulefiles/tasks/gaea/python_srw.lua | 4 ++-- modulefiles/tasks/gaea/run_vx.local.lua | 6 +++++- modulefiles/wflow_gaea.lua | 8 +++---- ush/machine/gaea.yaml | 21 ++++++++++--------- 7 files changed, 28 insertions(+), 23 deletions(-) create mode 100644 modulefiles/tasks/gaea/plot_allvars.local.lua diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh index ee7ed10603..3b587f1532 100644 --- a/etc/lmod-setup.csh +++ b/etc/lmod-setup.csh @@ -38,11 +38,7 @@ else if ( "$L_MACHINE" == singularity ) then module purge else if ( "$L_MACHINE" == gaea ) then - set ENV="/lustre/f2/dev/role.epic/contrib/apps/lmod/lmod/init/csh" - source $ENV - - setenv LMOD_SYSTEM_DEFAULT_MODULES "modules/3.2.11.4" - module --initial_load --no_redirect restore + source /lustre/f2/dev/role.epic/contrib/Lmod_init.csh else if ( "$L_MACHINE" == odin ) then module unload modules diff --git a/modulefiles/build_gaea_intel.lua b/modulefiles/build_gaea_intel.lua index 3bc053f315..4e98812727 100644 --- a/modulefiles/build_gaea_intel.lua +++ b/modulefiles/build_gaea_intel.lua @@ -1,6 +1,6 @@ help([[ This module loads libraries for building the UFS SRW App on -the NOAA RDHPC machine Gaea using Intel-2022.1.2 +the NOAA RDHPC machine Gaea using Intel-2022.0.2 ]]) whatis([===[Loads libraries needed for building the UFS SRW App on Gaea ]===]) diff --git a/modulefiles/tasks/gaea/plot_allvars.local.lua b/modulefiles/tasks/gaea/plot_allvars.local.lua new file mode 100644 index 0000000000..5cd4d13325 --- /dev/null +++ b/modulefiles/tasks/gaea/plot_allvars.local.lua @@ -0,0 +1,4 @@ +prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "regional_workflow") diff --git a/modulefiles/tasks/gaea/python_srw.lua b/modulefiles/tasks/gaea/python_srw.lua index 74336ca1cf..a18856b8d2 100644 --- a/modulefiles/tasks/gaea/python_srw.lua +++ b/modulefiles/tasks/gaea/python_srw.lua @@ -1,4 +1,4 @@ -prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/modulefiles") +prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) -setenv("SRW_ENV", "regional_workflow") +setenv("SRW_ENV", "workflow_tools") diff --git a/modulefiles/tasks/gaea/run_vx.local.lua b/modulefiles/tasks/gaea/run_vx.local.lua index 9833c539aa..d6d71abaf7 100644 --- a/modulefiles/tasks/gaea/run_vx.local.lua +++ b/modulefiles/tasks/gaea/run_vx.local.lua @@ -1,2 +1,6 @@ +--[[ +Compiler-specific modules are used for met and metplus libraries +--]] +load(pathJoin("met", os.getenv("met_ver") or "11.0.2")) +load(pathJoin("metplus", os.getenv("metplus_ver") or "5.0.2")) load("python_srw") -prepend_path("LD_LIBRARY_PATH", "/ncrc/sw/gaea-cle7/python/3.9/anaconda-base/envs/noaa_py3.9/lib") diff --git a/modulefiles/wflow_gaea.lua b/modulefiles/wflow_gaea.lua index e4b726aac1..719283224c 100644 --- a/modulefiles/wflow_gaea.lua +++ b/modulefiles/wflow_gaea.lua @@ -5,15 +5,15 @@ the NOAA RDHPC machine Gaea whatis([===[Loads libraries needed for running the UFS SRW App on gaea ]===]) -prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/modulefiles") +load("set_pythonpath") +prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/miniconda3/modulefiles") +prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/rocoto/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) load("rocoto") load("alps") -setenv("PROJ_LIB", "/lustre/f2/dev/role.epic/contrib/miniconda3/4.12.0/envs/regional_workflow/share/proj") - if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda: - > conda activate regional_workflow + > conda activate workflow_tools ]===]) end diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index 9653d74c45..ddd3c05676 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -5,20 +5,21 @@ platform: CCPA_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc MRMS_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc NDAS_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc - METPLUS_PATH: /usw/met/METplus/METplus-4.1.3 + METPLUS_PATH: /lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/intel-classic-2022.0.2/metplus/5.0.2 MET_BIN_EXEC: bin - MET_INSTALL_DIR: /usw/met/10.1.2 + MET_INSTALL_DIR: /lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/intel-classic-2022.0.2/met/11.0.2 DOMAIN_PREGEN_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen - QUEUE_DEFAULT: normal - QUEUE_FCST: normal - QUEUE_HPSS: normal + QUEUE_DEFAULT: windfall + QUEUE_FCST: windfall + QUEUE_HPSS: windfall REMOVE_MEMORY: True - RUN_CMD_FCST: srun --export=ALL --mpi=pmi2 -n ${PE_MEMBER01} - RUN_CMD_POST: srun --export=ALL --mpi=pmi2 -n $nprocs - RUN_CMD_PRDGEN: srun --export=ALL --mpi=pmi2 -n $nprocs + RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01} + RUN_CMD_POST: srun --export=ALL -n $nprocs + RUN_CMD_PRDGEN: srun --export=ALL -n $nprocs RUN_CMD_SERIAL: time - RUN_CMD_UTILS: srun --export=ALL --mpi=pmi2 -n $nprocs - SCHED_NATIVE_CMD: -M c3 --export=NONE + RUN_CMD_UTILS: srun --export=ALL -n $nprocs + SCHED_NATIVE_CMD: --clusters=c4 --export=NONE + SCHED_NATIVE_CMD_HPSS: --clusters=c3,c4 --export=NONE PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data TEST_PREGEN_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen From e54c8f5efee1ad171f8fc34224a808f5a4d323f3 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Tue, 11 Jul 2023 03:17:11 +0000 Subject: [PATCH 03/13] load alps module for Gaea --- modulefiles/build_gaea_intel.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/build_gaea_intel.lua b/modulefiles/build_gaea_intel.lua index 4e98812727..a9777d42ad 100644 --- a/modulefiles/build_gaea_intel.lua +++ b/modulefiles/build_gaea_intel.lua @@ -11,7 +11,7 @@ prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-clas load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2")) load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20")) -load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37")) +load("alps") load("srw_common") From e65cba65333d1e5dfc18d27ff13a7046cbc5b3a7 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Tue, 11 Jul 2023 17:38:02 -0400 Subject: [PATCH 04/13] updated version of ncdiag/1.1.1 for GSI build --- modulefiles/srw_common.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua index 7fa6ef9fea..12b5122c18 100644 --- a/modulefiles/srw_common.lua +++ b/modulefiles/srw_common.lua @@ -27,6 +27,6 @@ load("sigio/2.3.2") load("w3nco/2.4.1") load("wrf_io/1.2.0") -load("ncdiag/1.0.0") +load("ncdiag/1.1.1") load("ncio/1.1.2") load("wgrib2/2.0.8") From adec3be7c072fb31ac3621e939494f27d594b8f6 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Sat, 22 Jul 2023 10:32:03 -0400 Subject: [PATCH 05/13] Updating modulefiles for Gaea --- modulefiles/build_gaea_intel.lua | 10 +++++++++- modulefiles/wflow_gaea.lua | 6 +++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/modulefiles/build_gaea_intel.lua b/modulefiles/build_gaea_intel.lua index a9777d42ad..bb156952e3 100644 --- a/modulefiles/build_gaea_intel.lua +++ b/modulefiles/build_gaea_intel.lua @@ -11,9 +11,17 @@ prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-clas load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2")) load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20")) -load("alps") load("srw_common") +-- Need at runtime +load("alps") + +local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/" +prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64")) +pushenv("MKLROOT", MKLROOT) + +pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601") +pushenv("CRAYPE_LINK_TYPE","dynamic") setenv("CC","cc") setenv("FC","ftn") diff --git a/modulefiles/wflow_gaea.lua b/modulefiles/wflow_gaea.lua index 719283224c..a26f0e5abb 100644 --- a/modulefiles/wflow_gaea.lua +++ b/modulefiles/wflow_gaea.lua @@ -7,11 +7,15 @@ whatis([===[Loads libraries needed for running the UFS SRW App on gaea ]===]) load("set_pythonpath") prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/miniconda3/modulefiles") -prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/rocoto/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) +prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/rocoto/modulefiles") load("rocoto") load("alps") +pushenv("MKLROOT", "/opt/intel/oneapi/mkl/2022.0.2/") +pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601") +setenv("PMI_NO_PREINITIALIZE","1") + if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda: > conda activate workflow_tools From 24ffba55b9d47bdc6df94bb81520972d224dd2c1 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Sat, 22 Jul 2023 11:02:56 -0400 Subject: [PATCH 06/13] updates for allowing use of met, metplus modules --- modulefiles/tasks/gaea/run_vx.local.lua | 4 ++-- ush/machine/gaea.yaml | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/modulefiles/tasks/gaea/run_vx.local.lua b/modulefiles/tasks/gaea/run_vx.local.lua index d6d71abaf7..5979a8db96 100644 --- a/modulefiles/tasks/gaea/run_vx.local.lua +++ b/modulefiles/tasks/gaea/run_vx.local.lua @@ -1,6 +1,6 @@ --[[ Compiler-specific modules are used for met and metplus libraries --]] -load(pathJoin("met", os.getenv("met_ver") or "11.0.2")) -load(pathJoin("metplus", os.getenv("metplus_ver") or "5.0.2")) +load(pathJoin("met", os.getenv("met_ver") or "10.1.2")) +load(pathJoin("metplus", os.getenv("metplus_ver") or "4.1.3")) load("python_srw") diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index baf35bd3fa..cb706676a3 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -5,9 +5,6 @@ platform: CCPA_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc MRMS_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc NDAS_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc - METPLUS_PATH: /lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/intel-classic-2022.0.2/metplus/5.0.2 - MET_BIN_EXEC: bin - MET_INSTALL_DIR: /lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/intel-classic-2022.0.2/met/11.0.2 DOMAIN_PREGEN_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen QUEUE_DEFAULT: windfall QUEUE_FCST: windfall From 88ed69a4143e9051dc6dc6ce844fd90b17d116f6 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Sat, 22 Jul 2023 13:14:54 -0400 Subject: [PATCH 07/13] update met, metplus versions --- modulefiles/tasks/gaea/run_vx.local.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modulefiles/tasks/gaea/run_vx.local.lua b/modulefiles/tasks/gaea/run_vx.local.lua index 5979a8db96..d6d71abaf7 100644 --- a/modulefiles/tasks/gaea/run_vx.local.lua +++ b/modulefiles/tasks/gaea/run_vx.local.lua @@ -1,6 +1,6 @@ --[[ Compiler-specific modules are used for met and metplus libraries --]] -load(pathJoin("met", os.getenv("met_ver") or "10.1.2")) -load(pathJoin("metplus", os.getenv("metplus_ver") or "4.1.3")) +load(pathJoin("met", os.getenv("met_ver") or "11.0.2")) +load(pathJoin("metplus", os.getenv("metplus_ver") or "5.0.2")) load("python_srw") From 623115608d37eada0227138e9d05aba2ca3ecf24 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Fri, 4 Aug 2023 10:41:02 -0400 Subject: [PATCH 08/13] Updates for Gaea to enable MMet verification tasks --- modulefiles/tasks/gaea/run_vx.local.lua | 4 ++-- parm/metplus/common.conf | 2 +- ...ional_run_met_genensprod_or_ensemblestat.sh | 3 --- ...egional_run_met_gridstat_or_pointstat_vx.sh | 3 --- ...run_met_gridstat_or_pointstat_vx_ensmean.sh | 3 --- ...run_met_gridstat_or_pointstat_vx_ensprob.sh | 3 --- scripts/exregional_run_met_pb2nc_obs.sh | 3 --- scripts/exregional_run_met_pcpcombine.sh | 3 --- ...ensemble_verification_only_vx_time_lag.yaml | 5 ----- ush/config.community.yaml | 2 -- ush/config.da_cycling.yaml | 2 -- ush/config_defaults.yaml | 15 --------------- ush/machine/gaea.yaml | 18 ++++++++++++++++-- 13 files changed, 19 insertions(+), 47 deletions(-) diff --git a/modulefiles/tasks/gaea/run_vx.local.lua b/modulefiles/tasks/gaea/run_vx.local.lua index d6d71abaf7..5979a8db96 100644 --- a/modulefiles/tasks/gaea/run_vx.local.lua +++ b/modulefiles/tasks/gaea/run_vx.local.lua @@ -1,6 +1,6 @@ --[[ Compiler-specific modules are used for met and metplus libraries --]] -load(pathJoin("met", os.getenv("met_ver") or "11.0.2")) -load(pathJoin("metplus", os.getenv("metplus_ver") or "5.0.2")) +load(pathJoin("met", os.getenv("met_ver") or "10.1.2")) +load(pathJoin("metplus", os.getenv("metplus_ver") or "4.1.3")) load("python_srw") diff --git a/parm/metplus/common.conf b/parm/metplus/common.conf index 52a0dbdadc..bac1b6bc1a 100644 --- a/parm/metplus/common.conf +++ b/parm/metplus/common.conf @@ -6,7 +6,7 @@ METPLUS_BASE = {ENV[METPLUS_PATH]} # Met install location MET_INSTALL_DIR = {ENV[MET_INSTALL_DIR]} MET_BASE = {MET_INSTALL_DIR}/share/met -MET_BIN_DIR = {MET_INSTALL_DIR}/{ENV[MET_BIN_EXEC]} +MET_BIN_DIR = {MET_INSTALL_DIR}/bin # Location of METplus parm directory METPLUS_PARM_BASE = {ENV[METPLUS_CONF]} diff --git a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh index 033140c425..9ccb2d1b25 100755 --- a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh +++ b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh @@ -293,9 +293,6 @@ fi # #----------------------------------------------------------------------- # -export MET_INSTALL_DIR -export METPLUS_PATH -export MET_BIN_EXEC export METPLUS_CONF export LOGDIR # diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh index 73f934b796..46c93ee299 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh @@ -294,9 +294,6 @@ fi # #----------------------------------------------------------------------- # -export MET_INSTALL_DIR -export METPLUS_PATH -export MET_BIN_EXEC export METPLUS_CONF export LOGDIR # diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh index f2f916fa79..bf3a8280a1 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh @@ -256,9 +256,6 @@ ACCUM_NO_PAD=$( printf "%0d" "${ACCUM_HH}" ) # #----------------------------------------------------------------------- # -export MET_INSTALL_DIR -export METPLUS_PATH -export MET_BIN_EXEC export METPLUS_CONF export LOGDIR # diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh index 69a08d35e3..89c128eeed 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh @@ -212,9 +212,6 @@ ACCUM_NO_PAD=$( printf "%0d" "${ACCUM_HH}" ) # #----------------------------------------------------------------------- # -export MET_INSTALL_DIR -export METPLUS_PATH -export MET_BIN_EXEC export METPLUS_CONF export LOGDIR # diff --git a/scripts/exregional_run_met_pb2nc_obs.sh b/scripts/exregional_run_met_pb2nc_obs.sh index 3365e7f8c6..be0f0a2b91 100755 --- a/scripts/exregional_run_met_pb2nc_obs.sh +++ b/scripts/exregional_run_met_pb2nc_obs.sh @@ -161,9 +161,6 @@ fi # #----------------------------------------------------------------------- # -export MET_INSTALL_DIR -export METPLUS_PATH -export MET_BIN_EXEC export METPLUS_CONF export LOGDIR # diff --git a/scripts/exregional_run_met_pcpcombine.sh b/scripts/exregional_run_met_pcpcombine.sh index 187b953e53..cb047163cb 100755 --- a/scripts/exregional_run_met_pcpcombine.sh +++ b/scripts/exregional_run_met_pcpcombine.sh @@ -244,9 +244,6 @@ fi # #----------------------------------------------------------------------- # -export MET_INSTALL_DIR -export METPLUS_PATH -export MET_BIN_EXEC export METPLUS_CONF export LOGDIR # diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml index ded24b22f4..b38eef188b 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml @@ -31,11 +31,6 @@ global: NUM_ENS_MEMBERS: 2 ENS_TIME_LAG_HRS: '[ 0, 12 ]' -platform: - CCPA_OBS_DIR: '/scratch2/BMC/fv3lam/ens_design_RRFS/obs_data/ccpa/proc' - MRMS_OBS_DIR: '/scratch2/BMC/fv3lam/ens_design_RRFS/obs_data/mrms/proc' - NDAS_OBS_DIR: '/scratch2/BMC/fv3lam/ens_design_RRFS/obs_data/ndas/proc' - verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km VX_FCST_INPUT_BASEDIR: '{{ platform.get("TEST_VX_FCST_INPUT_BASEDIR") }}' diff --git a/ush/config.community.yaml b/ush/config.community.yaml index 565f8ec613..417b9edb91 100644 --- a/ush/config.community.yaml +++ b/ush/config.community.yaml @@ -6,8 +6,6 @@ user: MACHINE: hera ACCOUNT: an_account platform: - MET_INSTALL_DIR: "" - METPLUS_PATH: "" CCPA_OBS_DIR: "" MRMS_OBS_DIR: "" NDAS_OBS_DIR: "" diff --git a/ush/config.da_cycling.yaml b/ush/config.da_cycling.yaml index da8b1b8278..105a813f2c 100644 --- a/ush/config.da_cycling.yaml +++ b/ush/config.da_cycling.yaml @@ -6,8 +6,6 @@ user: MACHINE: hera ACCOUNT: an_account platform: - MET_INSTALL_DIR: "" - METPLUS_PATH: "" CCPA_OBS_DIR: "" MRMS_OBS_DIR: "" NDAS_OBS_DIR: "" diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 1e726f2fe0..7308b84af1 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -233,18 +233,6 @@ platform: # # Set METplus parameters. Definitions: # - # MET_INSTALL_DIR: - # Location to top-level directory of MET installation. - # - # MET_BIN_EXEC: - # Subdirectory containing MET binaries e.g. "bin" - # - # METPLUS_PATH: - # Location to top-level directory of METplus installation. - # - # MET_BIN_EXEC - # Name of subdirectory where METplus executables are installed. - # # CCPA_OBS_DIR: # User-specified location of top-level directory where CCPA hourly # precipitation files used by METplus are located. This parameter @@ -354,9 +342,6 @@ platform: # #----------------------------------------------------------------------- # - MET_INSTALL_DIR: "" - MET_BIN_EXEC: "" - METPLUS_PATH: "" CCPA_OBS_DIR: "" NOHRSC_OBS_DIR: "" MRMS_OBS_DIR: "" diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index cb706676a3..98ee88bec8 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -15,13 +15,14 @@ platform: RUN_CMD_PRDGEN: srun --export=ALL -n $nprocs RUN_CMD_SERIAL: time RUN_CMD_UTILS: srun --export=ALL --mpi=pmi2 -n $nprocs - SCHED_NATIVE_CMD: --clusters=c3,c4 --export=NONE - SCHED_NATIVE_CMD_HPSS: --clusters=c3,c4 --export=NONE + SCHED_NATIVE_CMD: --clusters=c4 --export=NONE + SCHED_NATIVE_CMD_HPSS: --clusters=c4 --export=NONE PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data TEST_PREGEN_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir + TEST_VX_FCST_INPUT_BASEDIR: '{{ "/lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/output_data/fcst_" }}{{ "ens" if (global.NUM_ENS_MEMBERS > 0) else "det" }}{{ "/{{workflow.PREDEF_GRID_NAME}}" }}{% raw %}{% endraw %}' FIXaer: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/fix/fix_aer FIXgsi: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/fix/fix_gsi FIXgsm: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/fix/fix_am @@ -29,8 +30,21 @@ platform: FIXorg: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/fix/fix_orog FIXsfc: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo FIXshp: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/NaturalEarth + EXTRN_MDL_DATA_STORES: aws disk data: ics_lbcs: FV3GFS: nemsio: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/nemsio/${yyyymmdd}${hh} grib2: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/grib2/${yyyymmdd}${hh} + RAP: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh} + HRRR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh} +rocoto: + tasks: + metatask_run_ensemble: + task_run_fcst_mem#mem#: + cores: '{{ task_run_fcst.PE_MEMBER01 // 1 }}' + native: '--cpus-per-task {{ task_run_fcst.OMP_NUM_THREADS_RUN_FCST|int }} --exclusive {{ platform.SCHED_NATIVE_CMD }}' + nodes: + nnodes: + nodesize: + ppn: From 5f81ccc12f6075dee9bde13799540611a6fc3558 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Thu, 10 Aug 2023 18:06:18 -0400 Subject: [PATCH 09/13] Machine file update for Gaea C4 --- ush/machine/gaea.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index 98ee88bec8..25cd60fe1e 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -10,13 +10,14 @@ platform: QUEUE_FCST: windfall QUEUE_HPSS: windfall REMOVE_MEMORY: True + PARTITION_HPSS: eslogin RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01} RUN_CMD_POST: srun --export=ALL -n $nprocs RUN_CMD_PRDGEN: srun --export=ALL -n $nprocs RUN_CMD_SERIAL: time RUN_CMD_UTILS: srun --export=ALL --mpi=pmi2 -n $nprocs SCHED_NATIVE_CMD: --clusters=c4 --export=NONE - SCHED_NATIVE_CMD_HPSS: --clusters=c4 --export=NONE + SCHED_NATIVE_CMD_HPSS: --clusters=es --export=NONE PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data TEST_PREGEN_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen @@ -30,7 +31,7 @@ platform: FIXorg: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/fix/fix_orog FIXsfc: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo FIXshp: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/NaturalEarth - EXTRN_MDL_DATA_STORES: aws disk + EXTRN_MDL_DATA_STORES: aws data: ics_lbcs: FV3GFS: From a300700e0ab1a6e9c3f3e14e2ef30306692b4c22 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Fri, 11 Aug 2023 15:48:59 -0400 Subject: [PATCH 10/13] Reactivate Gaea pipelines in Jenkinsfile and change from Jet to Jet-EPIC due to resource issues on Jet (requiring update to .cicd/scripts/srw_*.sh scripts). --- .cicd/Jenkinsfile | 14 +++++--------- .cicd/scripts/srw_build.sh | 4 ++++ .cicd/scripts/srw_ftest.sh | 6 +++++- .cicd/scripts/srw_test.sh | 4 ++++ 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 04bb2034da..8fa2c0116e 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -10,12 +10,10 @@ pipeline { parameters { // Allow job runner to filter based on platform // Use the line below to enable all PW clusters - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet-epic', 'orion', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') // Use the line below to enable the PW AWS cluster - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') - // Use the line below to re-enable Gaea - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use') - choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet-epic', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') + choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet-epic', 'orion'], description: 'Specify the platform(s) to use') // Allow job runner to filter based on compiler choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') // Uncomment the following line to re-enable comprehensive tests @@ -78,8 +76,7 @@ pipeline { axes { axis { name 'SRW_PLATFORM' - // values 'cheyenne', 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' - values 'cheyenne', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'cheyenne', 'gaea', 'hera', 'jet-epic', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { @@ -93,8 +90,7 @@ pipeline { exclude { axis { name 'SRW_PLATFORM' - // values 'gaea', 'jet', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' - values 'jet', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'gaea', 'jet-epic', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { diff --git a/.cicd/scripts/srw_build.sh b/.cicd/scripts/srw_build.sh index 7c315655ab..10573915ce 100755 --- a/.cicd/scripts/srw_build.sh +++ b/.cicd/scripts/srw_build.sh @@ -24,6 +24,10 @@ else platform="${SRW_PLATFORM}" fi +if [[ "${SRW_PLATFORM}" = jet-epic ]]; then + platform='jet' +fi + # Build and install cd ${workspace}/tests set +e diff --git a/.cicd/scripts/srw_ftest.sh b/.cicd/scripts/srw_ftest.sh index 4ed136dd78..572b7aa258 100755 --- a/.cicd/scripts/srw_ftest.sh +++ b/.cicd/scripts/srw_ftest.sh @@ -42,6 +42,10 @@ else platform="${SRW_PLATFORM}" fi +if [[ "${SRW_PLATFORM}" = jet-epic ]]; then + platform='jet' +fi + # Test directories we2e_experiment_base_dir="${workspace}/expt_dirs" we2e_test_dir="${workspace}/tests/WE2E" @@ -64,7 +68,7 @@ sed "s|^workflow:|workflow:\n EXPT_BASEDIR: ${workspace}/expt_dirs|1" -i ush/co sed "s|^workflow:|workflow:\n EXEC_SUBDIR: ${workspace}/install_${SRW_COMPILER}/exec|1" -i ush/config.yaml # DATA_LOCATION differs on each platform ... find it. -export DATA_LOCATION=$(grep TEST_EXTRN_MDL_SOURCE_BASEDIR ${workspace}/ush/machine/${SRW_PLATFORM,,}.yaml | awk '{printf "%s", $2}') +export DATA_LOCATION=$(grep TEST_EXTRN_MDL_SOURCE_BASEDIR ${workspace}/ush/machine/${platform,,}.yaml | awk '{printf "%s", $2}') echo "DATA_LOCATION=${DATA_LOCATION}" # Configure a default test ... diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh index 50b4d1b842..2b0041a847 100755 --- a/.cicd/scripts/srw_test.sh +++ b/.cicd/scripts/srw_test.sh @@ -25,6 +25,10 @@ else platform="${SRW_PLATFORM}" fi +if [[ "${SRW_PLATFORM}" = jet-epic ]]; then + platform='jet' +fi + # Test directories we2e_experiment_base_dir="${workspace}/expt_dirs" we2e_test_dir="${workspace}/tests/WE2E" From 747890b97b0f3061b8d977b914a97f03983e59d4 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Fri, 11 Aug 2023 17:08:25 -0400 Subject: [PATCH 11/13] Re-enable Functional Workflow Task Tests for Gaea. --- .cicd/scripts/srw_ftest.sh | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.cicd/scripts/srw_ftest.sh b/.cicd/scripts/srw_ftest.sh index 572b7aa258..a3f401e028 100755 --- a/.cicd/scripts/srw_ftest.sh +++ b/.cicd/scripts/srw_ftest.sh @@ -82,6 +82,7 @@ sed 's|^platform:|platform:\n EXTRN_MDL_DATA_STORES: disk|g' -i ush/config.yaml source etc/lmod-setup.sh ${platform,,} module use modulefiles module load build_${platform,,}_${SRW_COMPILER} +module unload python module load wflow_${platform,,} [[ ${FORGIVE_CONDA} == true ]] && set +e +u # Some platforms have incomplete python3 or conda support, but wouldn't necessarily block workflow tests @@ -133,18 +134,13 @@ rm -f ${results_file} status=0 # Limit to machines that are fully ready -deny_machines=( gaea ) -if [[ ${deny_machines[@]} =~ ${platform,,} ]] ; then - echo "# Deny ${platform} - incomplete configuration." | tee -a ${results_file} -else - echo "# Try ${platform} with the first few simple SRW tasks ..." | tee -a ${results_file} - for task in ${TASKS[@]:0:${TASK_DEPTH}} ; do - echo -n "./$task.sh ... " - ./$task.sh > $task-log.txt 2>&1 && echo "COMPLETE" || echo "FAIL rc=$(( status+=$? ))" - # stop at the first sign of trouble ... - [[ 0 != ${status} ]] && echo "$task: FAIL" >> ${results_file} && break || echo "$task: COMPLETE" >> ${results_file} - done -fi +echo "# Try ${platform} with the first few simple SRW tasks ..." | tee -a ${results_file} +for task in ${TASKS[@]:0:${TASK_DEPTH}} ; do + echo -n "./$task.sh ... " + ./$task.sh > $task-log.txt 2>&1 && echo "COMPLETE" || echo "FAIL rc=$(( status+=$? ))" + # stop at the first sign of trouble ... + [[ 0 != ${status} ]] && echo "$task: FAIL" >> ${results_file} && break || echo "$task: COMPLETE" >> ${results_file} +done # Set exit code to number of failures set +e From 8c9b560a82e025b3cb4dbec378fb8a2539e5315b Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Mon, 14 Aug 2023 10:52:32 -0400 Subject: [PATCH 12/13] unload standard python in wflow_gaea.lua on compute nodes --- modulefiles/wflow_gaea.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/modulefiles/wflow_gaea.lua b/modulefiles/wflow_gaea.lua index a26f0e5abb..007e2b1fcd 100644 --- a/modulefiles/wflow_gaea.lua +++ b/modulefiles/wflow_gaea.lua @@ -5,6 +5,7 @@ the NOAA RDHPC machine Gaea whatis([===[Loads libraries needed for running the UFS SRW App on gaea ]===]) +unload("python") load("set_pythonpath") prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) From 77aa6e3cb2f1b2a4032f9f2b14399d9236f442ec Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Mon, 14 Aug 2023 14:58:42 +0000 Subject: [PATCH 13/13] Remove module unload python from .cicd/scripts/srw_ftest.sh. --- .cicd/scripts/srw_ftest.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.cicd/scripts/srw_ftest.sh b/.cicd/scripts/srw_ftest.sh index a3f401e028..28c67c32d4 100755 --- a/.cicd/scripts/srw_ftest.sh +++ b/.cicd/scripts/srw_ftest.sh @@ -82,7 +82,6 @@ sed 's|^platform:|platform:\n EXTRN_MDL_DATA_STORES: disk|g' -i ush/config.yaml source etc/lmod-setup.sh ${platform,,} module use modulefiles module load build_${platform,,}_${SRW_COMPILER} -module unload python module load wflow_${platform,,} [[ ${FORGIVE_CONDA} == true ]] && set +e +u # Some platforms have incomplete python3 or conda support, but wouldn't necessarily block workflow tests