From 2b080dfaa920efb2bb2c29a5f9eb1e83e492c805 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Tue, 12 Sep 2023 06:07:38 -0500 Subject: [PATCH 1/7] Adding Hercules as a Tier-1 platform --- etc/lmod-setup.csh | 2 +- etc/lmod-setup.sh | 2 +- modulefiles/build_hercules_intel.lua | 31 +++++++++++++++ modulefiles/srw_common.lua | 22 +++++------ modulefiles/tasks/hercules/aqm_ics.local.lua | 2 + modulefiles/tasks/hercules/aqm_lbcs.local.lua | 1 + .../tasks/hercules/fire_emission.local.lua | 2 + .../miniconda_regional_workflow_cmaq.lua | 4 ++ .../tasks/hercules/nexus_emission.local.lua | 2 + .../tasks/hercules/nexus_post_split.local.lua | 2 + .../tasks/hercules/plot_allvars.local.lua | 5 +++ .../tasks/hercules/point_source.local.lua | 1 + modulefiles/tasks/hercules/python_srw.lua | 5 +++ modulefiles/tasks/hercules/run_vx.local.lua | 6 +++ .../miniconda_regional_workflow_cmaq.lua | 2 +- .../tasks/orion/plot_allvars.local.lua | 2 +- modulefiles/wflow_hercules.lua | 21 ++++++++++ ush/machine/hercules.yaml | 39 +++++++++++++++++++ ush/valid_param_vals.yaml | 2 +- 19 files changed, 137 insertions(+), 16 deletions(-) create mode 100644 modulefiles/build_hercules_intel.lua create mode 100644 modulefiles/tasks/hercules/aqm_ics.local.lua create mode 100644 modulefiles/tasks/hercules/aqm_lbcs.local.lua create mode 100644 modulefiles/tasks/hercules/fire_emission.local.lua create mode 100644 modulefiles/tasks/hercules/miniconda_regional_workflow_cmaq.lua create mode 100644 modulefiles/tasks/hercules/nexus_emission.local.lua create mode 100644 modulefiles/tasks/hercules/nexus_post_split.local.lua create mode 100644 modulefiles/tasks/hercules/plot_allvars.local.lua create mode 100644 modulefiles/tasks/hercules/point_source.local.lua create mode 100644 modulefiles/tasks/hercules/python_srw.lua create mode 100644 modulefiles/tasks/hercules/run_vx.local.lua create mode 100644 modulefiles/wflow_hercules.lua create mode 100644 ush/machine/hercules.yaml diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh index 3b587f1532..bd62dc5833 100644 --- a/etc/lmod-setup.csh +++ b/etc/lmod-setup.csh @@ -6,7 +6,7 @@ Usage: source etc/lmod-setup.csh PLATFORM OPTIONS: PLATFORM - name of machine you are building on - (e.g. cheyenne | hera | jet | orion | wcoss2 ) + (e.g. cheyenne | hera | jet | orion | hercules | wcoss2 ) EOF_USAGE exit 1 else diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh index 9c330d910c..3a151da1eb 100644 --- a/etc/lmod-setup.sh +++ b/etc/lmod-setup.sh @@ -7,7 +7,7 @@ Usage: source etc/lmod-setup.sh PLATFORM OPTIONS: PLATFORM - name of machine you are building on - (e.g. cheyenne | hera | jet | orion | wcoss2 ) + (e.g. cheyenne | hera | jet | orion | hercules | wcoss2 ) EOF_USAGE exit 1 else diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua new file mode 100644 index 0000000000..e685661812 --- /dev/null +++ b/modulefiles/build_hercules_intel.lua @@ -0,0 +1,31 @@ +help([[ +This module loads libraries for building the UFS SRW App on +the MSU machine Hercules using intel-oneapi-compilers/2022.2.1 +]]) + +whatis([===[Loads libraries needed for building the UFS SRW App on Orion ]===]) + +load("contrib") +load("noaatools") + +load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.2")) + +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/hpc-stack/intel-2022.2.1/modulefiles/stack") +load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) +load(pathJoin("hpc-intel-oneapi-compilers", os.getenv("hpc_intel_ver") or "2022.2.1")) +load(pathJoin("hpc-intel-oneapi-mpi", os.getenv("hpc_mpi_ver") or "2021.7.1")) + +load("srw_common") + +load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.8.9.0")) +load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6")) + +-- setenv-march=core-avx2 +setenv("CFLAGS","-diag-disable=10441") +setenv("FFLAGS","-diag-disable=10441") + +setenv("CMAKE_C_COMPILER","mpiicc") +setenv("CMAKE_CXX_COMPILER","mpiicpc") +setenv("CMAKE_Fortran_COMPILER","mpiifort") +setenv("CMAKE_Platform","hercules.intel") + diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua index 12b5122c18..0dac50347d 100644 --- a/modulefiles/srw_common.lua +++ b/modulefiles/srw_common.lua @@ -1,17 +1,17 @@ -load("jasper/2.0.25") -load("zlib/1.2.11") -load_any("png/1.6.35", "libpng/1.6.37") +load_any("jasper/2.0.25","jasper/2.0.32") +load_any("zlib/1.2.11","zlib/1.2.13") +load("libpng/1.6.37") -load_any("netcdf/4.7.4", "netcdf-c/4.7.4") -load_any("netcdf/4.7.4", "netcdf-fortran/4.5.4") -load_any("pio/2.5.7", "parallelio/2.5.2") -load_any("esmf/8.3.0b09", "esmf/8.2.0") -load("fms/2022.04") +load_any("netcdf/4.9.2", "netcdf-c/4.9.2") +load_any("netcdf/4.9.2", "netcdf-fortran/4.6.0") +load_any("pio/2.5.10", "parallelio/2.5.9") +load("esmf/8.4.2") +load("fms/2023.01") load("bufr/11.7.0") load("bacio/2.4.1") -load("crtm/2.4.0") load("g2/3.4.5") +load("crtm/2.4.0") load("g2tmpl/1.10.2") load("ip/3.3.3") load("sp/2.3.3") @@ -19,13 +19,13 @@ load("w3emc/2.9.2") load_any("gftl-shared/v1.5.0", "gftl-shared/1.5.0") load_any("yafyaml/v0.5.1", "yafyaml/0.5.1") -load_any("mapl/2.22.0-esmf-8.3.0b09", "mapl/2.11.0-esmf-8.2.0") +load("mapl/2.35.2-esmf-8.4.2") load("nemsio/2.5.4") load("sfcio/1.4.1") load("sigio/2.3.2") load("w3nco/2.4.1") -load("wrf_io/1.2.0") +load_any("wrf_io/1.2.0","wrf-io/1.2.0") load("ncdiag/1.1.1") load("ncio/1.1.2") diff --git a/modulefiles/tasks/hercules/aqm_ics.local.lua b/modulefiles/tasks/hercules/aqm_ics.local.lua new file mode 100644 index 0000000000..5a7b0cece6 --- /dev/null +++ b/modulefiles/tasks/hercules/aqm_ics.local.lua @@ -0,0 +1,2 @@ +load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/hercules/aqm_lbcs.local.lua b/modulefiles/tasks/hercules/aqm_lbcs.local.lua new file mode 100644 index 0000000000..23370a8d60 --- /dev/null +++ b/modulefiles/tasks/hercules/aqm_lbcs.local.lua @@ -0,0 +1 @@ +load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) diff --git a/modulefiles/tasks/hercules/fire_emission.local.lua b/modulefiles/tasks/hercules/fire_emission.local.lua new file mode 100644 index 0000000000..5a7b0cece6 --- /dev/null +++ b/modulefiles/tasks/hercules/fire_emission.local.lua @@ -0,0 +1,2 @@ +load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/hercules/miniconda_regional_workflow_cmaq.lua b/modulefiles/tasks/hercules/miniconda_regional_workflow_cmaq.lua new file mode 100644 index 0000000000..c785373386 --- /dev/null +++ b/modulefiles/tasks/hercules/miniconda_regional_workflow_cmaq.lua @@ -0,0 +1,4 @@ +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "regional_workflow_cmaq") diff --git a/modulefiles/tasks/hercules/nexus_emission.local.lua b/modulefiles/tasks/hercules/nexus_emission.local.lua new file mode 100644 index 0000000000..5a7b0cece6 --- /dev/null +++ b/modulefiles/tasks/hercules/nexus_emission.local.lua @@ -0,0 +1,2 @@ +load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/hercules/nexus_post_split.local.lua b/modulefiles/tasks/hercules/nexus_post_split.local.lua new file mode 100644 index 0000000000..5a7b0cece6 --- /dev/null +++ b/modulefiles/tasks/hercules/nexus_post_split.local.lua @@ -0,0 +1,2 @@ +load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/hercules/plot_allvars.local.lua b/modulefiles/tasks/hercules/plot_allvars.local.lua new file mode 100644 index 0000000000..c995f83bb8 --- /dev/null +++ b/modulefiles/tasks/hercules/plot_allvars.local.lua @@ -0,0 +1,5 @@ +unload("python") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "regional_workflow") diff --git a/modulefiles/tasks/hercules/point_source.local.lua b/modulefiles/tasks/hercules/point_source.local.lua new file mode 100644 index 0000000000..07cacc8ce2 --- /dev/null +++ b/modulefiles/tasks/hercules/point_source.local.lua @@ -0,0 +1 @@ +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/hercules/python_srw.lua b/modulefiles/tasks/hercules/python_srw.lua new file mode 100644 index 0000000000..157e6884ea --- /dev/null +++ b/modulefiles/tasks/hercules/python_srw.lua @@ -0,0 +1,5 @@ +unload("python") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "workflow_tools") diff --git a/modulefiles/tasks/hercules/run_vx.local.lua b/modulefiles/tasks/hercules/run_vx.local.lua new file mode 100644 index 0000000000..5979a8db96 --- /dev/null +++ b/modulefiles/tasks/hercules/run_vx.local.lua @@ -0,0 +1,6 @@ +--[[ +Compiler-specific modules are used for met and metplus libraries +--]] +load(pathJoin("met", os.getenv("met_ver") or "10.1.2")) +load(pathJoin("metplus", os.getenv("metplus_ver") or "4.1.3")) +load("python_srw") diff --git a/modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua b/modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua index 9d84332bd5..890f642185 100644 --- a/modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua +++ b/modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua @@ -1,4 +1,4 @@ -prepend_path("MODULEPATH","/work/noaa/epic-ps/role-epic-ps/miniconda3/modulefiles") +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) setenv("SRW_ENV", "regional_workflow_cmaq") diff --git a/modulefiles/tasks/orion/plot_allvars.local.lua b/modulefiles/tasks/orion/plot_allvars.local.lua index 5b7b0afc57..94fed189be 100644 --- a/modulefiles/tasks/orion/plot_allvars.local.lua +++ b/modulefiles/tasks/orion/plot_allvars.local.lua @@ -1,5 +1,5 @@ unload("python") -append_path("MODULEPATH","/work/noaa/epic-ps/role-epic-ps/miniconda3/modulefiles") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) setenv("SRW_ENV", "regional_workflow") diff --git a/modulefiles/wflow_hercules.lua b/modulefiles/wflow_hercules.lua new file mode 100644 index 0000000000..471b1f8919 --- /dev/null +++ b/modulefiles/wflow_hercules.lua @@ -0,0 +1,21 @@ +help([[ +This module loads python environement for running SRW on +the MSU machine Hercules +]]) + +whatis([===[Loads libraries needed for running SRW on Hercules ]===]) + +load("contrib") +load("rocoto") +load("set_pythonpath") + +unload("python") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +if mode() == "load" then + LmodMsgRaw([===[Please do the following to activate conda: + > conda activate workflow_tools +]===]) +end + diff --git a/ush/machine/hercules.yaml b/ush/machine/hercules.yaml new file mode 100644 index 0000000000..74b87e4a5e --- /dev/null +++ b/ush/machine/hercules.yaml @@ -0,0 +1,39 @@ +platform: + WORKFLOW_MANAGER: rocoto + NCORES_PER_NODE: 80 + SCHED: slurm + TEST_CCPA_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc + DOMAIN_PREGEN_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen + PARTITION_DEFAULT: hercules + QUEUE_DEFAULT: batch + PARTITION_FCST: hercules + QUEUE_FCST: batch + PARTITION_HPSS: service + QUEUE_HPSS: batch + RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01} + RUN_CMD_POST: srun --export=ALL + RUN_CMD_PRDGEN: srun --export=ALL + RUN_CMD_SERIAL: time + RUN_CMD_UTILS: srun --export=ALL + RUN_CMD_NEXUS: srun --export=ALL + RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS} + SCHED_NATIVE_CMD: --export=NONE + PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' + TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data + TEST_AQM_INPUT_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/aqm_data + TEST_PREGEN_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen + TEST_VX_FCST_INPUT_BASEDIR: '{{ "/work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/output_data/fcst_" }}{{ "ens" if (global.NUM_ENS_MEMBERS > 0) else "det" }}{{ "/{{workflow.PREDEF_GRID_NAME}}" }}{% raw %}{% endraw %}' + FIXaer: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_aer + FIXgsi: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_gsi + FIXgsm: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_am + FIXlut: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_lut + FIXorg: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_orog + FIXsfc: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo + FIXshp: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/NaturalEarth + EXTRN_MDL_DATA_STORES: aws nomads +task_process_bufr: + OBSPATH_TEMPLATE: '${OBSPATH}/${YYYYMMDD}.rap/rap' +task_analysis_gsi: + OBSPATH_TEMPLATE: '${OBSPATH}/${YYYYMMDD}.rap/rap' diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index b838e5b95c..912270afa1 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -4,7 +4,7 @@ valid_vals_RUN_ENVIR: ["nco", "community"] valid_vals_VERBOSE: [True, False] valid_vals_DEBUG: [True, False] -valid_vals_MACHINE: ["HERA", "WCOSS2", "ORION", "JET", "ODIN", "CHEYENNE", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"] +valid_vals_MACHINE: ["HERA", "WCOSS2", "ORION", "HERCULES", "JET", "ODIN", "CHEYENNE", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"] valid_vals_SCHED: ["slurm", "pbspro", "lsf", "lsfcray", "none"] valid_vals_FCST_MODEL: ["ufs-weather-model"] valid_vals_WORKFLOW_MANAGER: ["rocoto", "ecflow", "none"] From 5c14fdbe1d3ad79b5f49c4c2fb97998f370181eb Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Fri, 15 Sep 2023 17:09:59 -0500 Subject: [PATCH 2/7] update build_hercules_intel.lua --- modulefiles/build_hercules_intel.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua index e685661812..8c0b4cb20b 100644 --- a/modulefiles/build_hercules_intel.lua +++ b/modulefiles/build_hercules_intel.lua @@ -20,7 +20,6 @@ load("srw_common") load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.8.9.0")) load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6")) --- setenv-march=core-avx2 setenv("CFLAGS","-diag-disable=10441") setenv("FFLAGS","-diag-disable=10441") From 9dde8a6749e3fade46c260e8ef15766ecbf7a5b2 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Sat, 16 Sep 2023 22:46:30 -0500 Subject: [PATCH 3/7] update modulefile and machine file for Hercules --- .cicd/Jenkinsfile | 14 +++++++------- modulefiles/build_hercules_intel.lua | 2 +- ush/machine/hercules.yaml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 007815edd2..660423c3c3 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -10,11 +10,11 @@ pipeline { parameters { // Allow job runner to filter based on platform // Use the line below to enable all PW clusters - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') // Use the line below to enable the PW AWS cluster - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use') - choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') + choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') // Allow job runner to filter based on compiler choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') // Uncomment the following line to re-enable comprehensive tests @@ -77,8 +77,8 @@ pipeline { axes { axis { name 'SRW_PLATFORM' - // values 'cheyenne', 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' - values 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + // values 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules'//, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'gaea', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { @@ -92,7 +92,7 @@ pipeline { exclude { axis { name 'SRW_PLATFORM' - values 'gaea', 'jet', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'gaea', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua index 8c0b4cb20b..1073952dc5 100644 --- a/modulefiles/build_hercules_intel.lua +++ b/modulefiles/build_hercules_intel.lua @@ -8,7 +8,7 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Orion ]===]) load("contrib") load("noaatools") -load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.2")) +load(pathJoin("cmake", os.getenv("cmake_ver") or "3.26.3")) prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/hpc-stack/intel-2022.2.1/modulefiles/stack") load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) diff --git a/ush/machine/hercules.yaml b/ush/machine/hercules.yaml index 74b87e4a5e..4fa3fe0fd3 100644 --- a/ush/machine/hercules.yaml +++ b/ush/machine/hercules.yaml @@ -16,7 +16,7 @@ platform: RUN_CMD_POST: srun --export=ALL RUN_CMD_PRDGEN: srun --export=ALL RUN_CMD_SERIAL: time - RUN_CMD_UTILS: srun --export=ALL + RUN_CMD_UTILS: srun --export=ALL -n $nprocs RUN_CMD_NEXUS: srun --export=ALL RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS} SCHED_NATIVE_CMD: --export=NONE From ac5e9b2536d5c6fca7ff07b264a345111483f22c Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Sat, 16 Sep 2023 23:41:11 -0500 Subject: [PATCH 4/7] add comprehensive.hercules test suite --- tests/WE2E/machine_suites/comprehensive.hercules | 1 + 1 file changed, 1 insertion(+) create mode 120000 tests/WE2E/machine_suites/comprehensive.hercules diff --git a/tests/WE2E/machine_suites/comprehensive.hercules b/tests/WE2E/machine_suites/comprehensive.hercules new file mode 120000 index 0000000000..18a56d9739 --- /dev/null +++ b/tests/WE2E/machine_suites/comprehensive.hercules @@ -0,0 +1 @@ +comprehensive.orion \ No newline at end of file From 21ca4ae43d24307a686bf52cee32a7c1d28ff1ef Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Sun, 17 Sep 2023 07:58:01 -0500 Subject: [PATCH 5/7] update standard data locations in hercules.yaml --- ush/machine/hercules.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ush/machine/hercules.yaml b/ush/machine/hercules.yaml index 4fa3fe0fd3..33d5732511 100644 --- a/ush/machine/hercules.yaml +++ b/ush/machine/hercules.yaml @@ -32,8 +32,14 @@ platform: FIXorg: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_orog FIXsfc: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo FIXshp: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/NaturalEarth - EXTRN_MDL_DATA_STORES: aws nomads -task_process_bufr: - OBSPATH_TEMPLATE: '${OBSPATH}/${YYYYMMDD}.rap/rap' -task_analysis_gsi: - OBSPATH_TEMPLATE: '${OBSPATH}/${YYYYMMDD}.rap/rap' + EXTRN_MDL_DATA_STORES: aws +data: + ics_lbcs: + FV3GFS: + nemsio: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/nemsio/${yyyymmdd}${hh} + grib2: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/grib2/${yyyymmdd}${hh} + netcdf: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/netcdf/${yyyymmdd}${hh} + NAM: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/NAM/${yyyymmdd}${hh} + HRRR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh} + RAP: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh} + GSMGFS: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/GSMGFS/${yyyymmdd}${hh} From bc6e4798bf5ba47de9820ef42babaeada28532b1 Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Thu, 21 Sep 2023 11:25:52 -0400 Subject: [PATCH 6/7] Rename coverage.cheyenne.gnu to coverage.hercules --- .../machine_suites/{coverage.cheyenne.gnu => coverage.hercules} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/WE2E/machine_suites/{coverage.cheyenne.gnu => coverage.hercules} (100%) diff --git a/tests/WE2E/machine_suites/coverage.cheyenne.gnu b/tests/WE2E/machine_suites/coverage.hercules similarity index 100% rename from tests/WE2E/machine_suites/coverage.cheyenne.gnu rename to tests/WE2E/machine_suites/coverage.hercules From 48f3d3989596279612ffb171a25781adb483709b Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Thu, 21 Sep 2023 17:14:52 -0400 Subject: [PATCH 7/7] Update build.sh --- tests/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/build.sh b/tests/build.sh index 39c54fa509..c6cbe89c1d 100755 --- a/tests/build.sh +++ b/tests/build.sh @@ -21,7 +21,7 @@ function usage() { exit 1 } -machines=( hera jet cheyenne derecho orion wcoss2 gaea odin singularity macos noaacloud ) +machines=( hera jet cheyenne derecho orion hercules wcoss2 gaea odin singularity macos noaacloud ) [[ $# -gt 4 ]] && usage