diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 1ebf80966c..32d71f1220 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -13,6 +13,7 @@ - [ ] WCOSS2 - [ ] Hera - [ ] Orion +- [ ] Hercules - [ ] Jet - Test cases: diff --git a/README.md b/README.md index 42ce155293..7066769c10 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ The above command is equal to: ./manage_externals/checkout_externals ./app_build.sh -p=[machine] ``` -where `[machine]` is `wcoss2`, `hera`, `jet`, or `orion`. +where `[machine]` is `wcoss2`, `hera`, `jet`, `orion`, or `hercules`. 4. Move to the home directory (rrfs-workflow): ``` diff --git a/doc/UsersGuide/source/Quickstart.rst b/doc/UsersGuide/source/Quickstart.rst index 4d96426877..108bb5d2fd 100644 --- a/doc/UsersGuide/source/Quickstart.rst +++ b/doc/UsersGuide/source/Quickstart.rst @@ -37,7 +37,7 @@ Building RRFS workflow ./app_build.sh --platform= - where ```` is ``wcoss2``, ``hera``, ``jet``, or ``orion``. + where ```` is ``wcoss2``, ``hera``, ``jet``, ``orion``, or ``hercules``. #. Move to the home directory (rrfs-workflow): @@ -58,7 +58,7 @@ Engineering Test: Non-DA module use modulefiles module load wflow_wcoss2 - * On Hera | Jet | Orion: + * On Hera | Jet | Orion | Hercules: .. code-block:: console @@ -66,7 +66,7 @@ Engineering Test: Non-DA module load wflow_ conda activate workflow_tools - where ```` is ``hera``, ``jet``, or ``orion``. + where ```` is ``hera``, ``jet``, ``orion``, or ``hercules``. #. Copy the pre-defined configuration file: @@ -75,7 +75,7 @@ Engineering Test: Non-DA cd ush cp sample_configs/non-DA_eng/config.nonDA...sh config.sh - where ```` is ``grib2`` or ``netcdf``, and ```` is ``wcoss2``, ``hera``, ``jet``, or ``orion``. Note that you may need to change ``ACCOUNT``, ``STMP``, or ``PTMP`` in the configuration file ``config.sh``. + where ```` is ``grib2`` or ``netcdf``, and ```` is ``wcoss2``, ``hera``, ``jet``, ``orion``, or ``hercules``. Note that you may need to change ``ACCOUNT``, ``STMP``, or ``PTMP`` in the configuration file ``config.sh``. #. Generate the experiment workflow: @@ -112,7 +112,7 @@ Engineering Test: DA module use modulefiles module load wflow_wcoss2 - * On Hera | Jet | Orion: + * On Hera | Jet | Orion | Hercules : .. code-block:: console @@ -120,7 +120,7 @@ Engineering Test: DA module load wflow_ conda activate workflow_tools - where ```` is ``hera``, ``jet``, or ``orion``. + where ```` is ``hera``, ``jet``, ``orion``, or ``hercules``. #. Copy the pre-defined configuration file: diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua new file mode 100644 index 0000000000..8482cfdaeb --- /dev/null +++ b/modulefiles/build_hercules_intel.lua @@ -0,0 +1,65 @@ +help([[ +This module loads libraries for building the RRFS workflow on +the MSU machine Hercules using Intel-2021.9.0 +]]) + +whatis([===[Loads libraries needed for building the RRFS worfklow on Hercules ]===]) + +load("contrib") +load("noaatools") + +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") +load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2021.9.0")) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver") or "2021.9.0")) +load("intel-oneapi-mkl/2022.2.1") +load(pathJoin("cmake", os.getenv("cmake_ver") or "3.23.1")) + +load(pathJoin("jasper", os.getenv("jasper_ver") or "2.0.32")) +load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37")) +load(pathJoin("parallelio", os.getenv("pio_ver") or "2.5.10")) +--loading parallelio will load netcdf_c, netcdf_fortran, hdf5, zlib, etc +load(pathJoin("esmf", os.getenv("esmf_ver") or "8.4.2")) +load(pathJoin("fms", os.getenv("fms_ver") or "2023.01")) + +load(pathJoin("bacio", os.getenv("bacio_ver") or "2.4.1")) +load(pathJoin("crtm", os.getenv("crtm_ver") or "2.4.0")) +load(pathJoin("g2", os.getenv("g2_ver") or "3.4.5")) +load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver") or "1.10.2")) +load(pathJoin("ip", os.getenv("ip_ver") or "4.3.0")) +load(pathJoin("sp", os.getenv("sp_ver") or "2.3.3")) + +load(pathJoin("gftl-shared", os.getenv("gftl-shared_ver") or "1.5.0")) +--load(pathJoin("yafyaml", os.getenv("yafyaml_ver") or "0.5.1")) +load(pathJoin("mapl", os.getenv("mapl_ver") or "2.35.2-esmf-8.4.2")) +load(pathJoin("scotch", os.getenv("scotch_ver") or "7.0.4")) + +--load(pathJoin("bufr", os.getenv("bufr_ver") or "12.0.0")) +load(pathJoin("gfsio", os.getenv("gfsio_ver") or "1.4.1")) +load(pathJoin("landsfcutil", os.getenv("landsfcutil_ver") or "2.4.1")) +load(pathJoin("nemsiogfs", os.getenv("nemsiogfs_ver") or "2.5.3")) +load(pathJoin("sigio", os.getenv("sigio_ver") or "2.3.2")) +load(pathJoin("sfcio", os.getenv("sfcio_ver") or "1.4.1")) +load(pathJoin("nemsio", os.getenv("nemsio_ver") or "2.5.4")) +--load(pathJoin("wrf_io", os.getenv("wrf_io_ver") or "1.2.0")) +load(pathJoin("ncio", os.getenv("ncio_ver") or "1.1.2")) +load(pathJoin("gsi-ncdiag", os.getenv("gsi-ncdiag_ver") or "1.1.1")) +load(pathJoin("w3emc", os.getenv("w3emc_ver") or "2.10.0")) +load(pathJoin("w3nco", os.getenv("w3nco_ver") or "2.4.1")) + +load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6")) +--load(pathJoin("prod_util", os.getenv("prod_util_ver") or "2.0.14")) +load(pathJoin("wgrib2", os.getenv("wgrib2_ver") or "3.1.1")) + +prepend_path("MODULEPATH", "/work/noaa/rtrr/gge/hercules/lua") +load("prod_util/2.0.15") +load("wrf_io/1.2.0a") +load("bufr/11.7.0a") +load("gsi-ncdiag/1.1.2a") + +unload("python/3.10.8") + +setenv("CMAKE_C_COMPILER","mpiicc") +setenv("CMAKE_CXX_COMPILER","mpiicpc") +setenv("CMAKE_Fortran_COMPILER","mpiifort") +setenv("CMAKE_Platform","hercules.intel") +setenv("I_MPI_EXTRA_FILESYSTEM","ON") diff --git a/modulefiles/tasks/hercules/python_srw.lua b/modulefiles/tasks/hercules/python_srw.lua new file mode 100644 index 0000000000..157e6884ea --- /dev/null +++ b/modulefiles/tasks/hercules/python_srw.lua @@ -0,0 +1,5 @@ +unload("python") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "workflow_tools") diff --git a/modulefiles/wflow_hercules.lua b/modulefiles/wflow_hercules.lua new file mode 100644 index 0000000000..1be5fcb351 --- /dev/null +++ b/modulefiles/wflow_hercules.lua @@ -0,0 +1,27 @@ +help([[ +This module loads python environement for running RRFS workflow on +the MSU machine Hercules +]]) + +whatis([===[Loads libraries needed for running RRFS workflow on Hercules ]===]) + +load("contrib") +load("rocoto") + +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") +load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2021.9.0")) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver") or "2021.9.0")) +load(pathJoin("crtm", os.getenv("crtm_ver") or "2.4.0")) + +load("wget") + +unload("python") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +if mode() == "load" then + LmodMsgRaw([===[Please do the following to activate conda: + > conda activate workflow_tools +]===]) +end + diff --git a/modulefiles/wflow_orion.lua b/modulefiles/wflow_orion.lua index 0d2a8593be..b08e03f2cd 100644 --- a/modulefiles/wflow_orion.lua +++ b/modulefiles/wflow_orion.lua @@ -9,10 +9,9 @@ load("contrib") load("rocoto") load("wget") -prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/hpc-stack/intel-2022.1.2/modulefiles/stack") -load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) -load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1.2")) -load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver") or "2022.1.2")) +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") +load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2022.0.2")) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver") or "2021.5.1")) load(pathJoin("crtm", os.getenv("crtm_ver") or "2.4.0")) unload("python") diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index a22fb1f636..cf0ced6716 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -745,7 +745,7 @@ MODULES_RUN_TASK_FP script. {%- elif machine in ["JET"] %} {{ extrn_mdl_sysbasedir_ics }}/@y@j@H00.gdas.t@Hz.atmf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc {{ extrn_mdl_sysbasedir_ics }}/@y@j@H00.gdas.t@Hz.sfcf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_ics }}/@y@j@H00.gdas.t@Hz.atmf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc {{ extrn_mdl_sysbasedir_ics }}/@y@j@H00.gdas.t@Hz.sfcf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc {%- endif %} @@ -757,7 +757,7 @@ MODULES_RUN_TASK_FP script. {{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }} {%- elif machine in ["HERA"] %} {{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }} - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }} {%- endif %} {%- endif %} @@ -866,7 +866,7 @@ MODULES_RUN_TASK_FP script. {%- elif machine in ["JET"] %} {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H00.gdas.t@Hz.atmf{{ "%03d" % h }}.#memNameGDAS#.nc {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H00.gdas.t@Hz.sfcf{{ "%03d" % h }}.#memNameGDAS#.nc - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H00.gdas.t@Hz.atmf{{ "%03d" % h }}.#memNameGDAS#.nc {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H00.gdas.t@Hz.sfcf{{ "%03d" % h }}.#memNameGDAS#.nc {%- endif %} @@ -875,7 +875,7 @@ MODULES_RUN_TASK_FP script. {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }} {%- elif machine in ["HERA"] %} {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }} - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }} {%- endif %} {%- endif %} @@ -919,7 +919,7 @@ MODULES_RUN_TASK_FP script. {%- for h in range(extrn_mdl_lbcs_offset_hrs, boundary_long_len_hrs+extrn_mdl_lbcs_offset_hrs+1, bc_update_interval) %} {%- if machine in ["WCOSS2"] %} {{ extrn_mdl_sysbasedir_lbcs }}/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.pgrb2.0p25.f{{ "%03d" % h }} - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_lbcs }}/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf{{ "%03d" % h }}.nc {%- else %} {%- if extrn_mdl_name_lbcs in ["GEFS"] %} diff --git a/parm/FV3LAM_wflow_nonDA.xml b/parm/FV3LAM_wflow_nonDA.xml index 0551ee712e..26ed8ec3fc 100644 --- a/parm/FV3LAM_wflow_nonDA.xml +++ b/parm/FV3LAM_wflow_nonDA.xml @@ -22,7 +22,7 @@ Parameters needed by the job scheduler. - +