diff --git a/.gitignore b/.gitignore index f8d1fc6f340..687404bef57 100644 --- a/.gitignore +++ b/.gitignore @@ -98,6 +98,7 @@ parm/regrid_sfc #-------------------------------------------- sorc/build.xml sorc/build.db +sorc/build.db-journal sorc/build_lock.db sorc/rocotostat.out sorc/*log @@ -173,6 +174,7 @@ ush/python/ufsda ush/python/soca ush/python/gen_bufr2ioda_json.py ush/python/gen_bufr2ioda_yaml.py +dev/ush/load_fv3gfs_modules.sh # ush log file ush/fetch-fix-data.log diff --git a/dev/ci/Jenkinsfile b/dev/ci/Jenkinsfile index 52a19c0e06f..af508d63427 100644 --- a/dev/ci/Jenkinsfile +++ b/dev/ci/Jenkinsfile @@ -6,9 +6,9 @@ def HOMEgfs_dev = 'none' def CI_CASES = '' def GH = 'none' // Map of the machine names (MACHINE_ID) to the Jenkins Node names -def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaeac5: 'GaeaC5', gaeac6: 'Gaeac6-EMC'] +def NodeName = [hera: 'Hera-EMC', ursa: 'Ursa-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaeac5: 'GaeaC5', gaeac6: 'Gaeac6-EMC'] // Location of the custom workspaces for each machine in the CI system. They are persistent for each iteration of the PR. -def custom_workspace = [hera: '/scratch3/NCEPDEV/global/glopara/CI/HERA', orion: '/work2/noaa/global/role-global/GFS_CI_CD/ORION/CI', hercules: '/work2/noaa/global/role-global/GFS_CI_CD/HERCULES/CI', gaeac5: '/gpfs/f5/epic/proj-shared/global/CI', gaeac6: '/gpfs/f6/drsa-precip3/world-shared/global/CI'] +def custom_workspace = [hera: '/scratch3/NCEPDEV/global/role.glopara/CI/HERA', ursa: '/scratch3/NCEPDEV/global/role.glopara/CI/URSA', orion: '/work2/noaa/global/role-global/GFS_CI_CD/ORION/CI', hercules: '/work2/noaa/global/role-global/GFS_CI_CD/HERCULES/CI', gaeac5: '/gpfs/f5/epic/proj-shared/global/CI', gaeac6: '/gpfs/f6/drsa-precip3/world-shared/global/CI'] def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' def STATUS = 'Passed' @@ -87,7 +87,7 @@ pipeline { Machine = machine[0].toUpperCase() + machine.substring(1) echo "Getting Common Workspace for ${Machine}" ws("${custom_workspace[machine]}/${env.CHANGE_ID}") { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'hercules-emc', 'hera-emc', 'orion-emc', 'gaeaC5', 'gaeaC6-emc'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'hercules-emc', 'hera-emc', 'ursa-emc', 'orion-emc', 'gaeaC5', 'gaeaC6-emc'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) GH = sh(script: "which gh || echo '~/bin/gh'", returnStdout: true).trim() CUSTOM_WORKSPACE = "${WORKSPACE}" HOMEgfs = "${CUSTOM_WORKSPACE}/global-workflow" diff --git a/dev/ci/cases/gcafsv1/C96_gcafs_cycled.yaml b/dev/ci/cases/gcafsv1/C96_gcafs_cycled.yaml index d8720d980e1..30c7208d7ec 100644 --- a/dev/ci/cases/gcafsv1/C96_gcafs_cycled.yaml +++ b/dev/ci/cases/gcafsv1/C96_gcafs_cycled.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 idate: 2021122012 edate: 2021122100 nens: 0 diff --git a/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA.yaml b/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA.yaml index a821c3b5e90..f1bb7596c40 100644 --- a/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA.yaml +++ b/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 idate: 2021122012 edate: 2021122100 nens: 0 diff --git a/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA_dev.yaml b/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA_dev.yaml index adb3516c3d2..7d3132d14c1 100644 --- a/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA_dev.yaml +++ b/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA_dev.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 idate: 2021122012 edate: 2021122200 nens: 0 diff --git a/dev/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/dev/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index abfbcd6092f..16c9b0e0c3e 100644 --- a/dev/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/dev/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -9,7 +9,7 @@ arguments: resdetocean: 5.0 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20250327 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20250808 idate: 2021032418 edate: 2021032500 nens: 0 diff --git a/dev/ci/cases/pr/C48mx500_hybAOWCDA.yaml b/dev/ci/cases/pr/C48mx500_hybAOWCDA.yaml index 238b1868e0e..93e078a1810 100644 --- a/dev/ci/cases/pr/C48mx500_hybAOWCDA.yaml +++ b/dev/ci/cases/pr/C48mx500_hybAOWCDA.yaml @@ -10,7 +10,7 @@ arguments: resdetocean: 5.0 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20250327 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20250808 idate: 2021032418 edate: 2021032500 nens: 2 diff --git a/dev/ci/cases/pr/C96C48_hybatmDA.yaml b/dev/ci/cases/pr/C96C48_hybatmDA.yaml index aab9b4e2dc6..87af2c367a1 100644 --- a/dev/ci/cases/pr/C96C48_hybatmDA.yaml +++ b/dev/ci/cases/pr/C96C48_hybatmDA.yaml @@ -10,7 +10,7 @@ arguments: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 idate: 2021122018 edate: 2021122106 nens: 2 diff --git a/dev/ci/cases/pr/C96C48_hybatmsnowDA.yaml b/dev/ci/cases/pr/C96C48_hybatmsnowDA.yaml index 89cbf7159ba..f22ed4dca12 100644 --- a/dev/ci/cases/pr/C96C48_hybatmsnowDA.yaml +++ b/dev/ci/cases/pr/C96C48_hybatmsnowDA.yaml @@ -9,7 +9,7 @@ arguments: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 idate: 2021122012 edate: 2021122100 nens: 2 diff --git a/dev/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/dev/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index 028c286e85c..28a3cd96201 100644 --- a/dev/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/dev/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -9,7 +9,7 @@ arguments: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 idate: 2024022318 edate: 2024022406 nens: 2 diff --git a/dev/ci/cases/pr/C96C48mx500_S2SW_cyc_gfs.yaml b/dev/ci/cases/pr/C96C48mx500_S2SW_cyc_gfs.yaml index ee457ab47c6..075e1f85b01 100644 --- a/dev/ci/cases/pr/C96C48mx500_S2SW_cyc_gfs.yaml +++ b/dev/ci/cases/pr/C96C48mx500_S2SW_cyc_gfs.yaml @@ -10,7 +10,7 @@ arguments: resdetocean: 5.0 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 idate: 2021122012 edate: 2021122100 nens: 2 diff --git a/dev/ci/cases/pr/C96_atm3DVar.yaml b/dev/ci/cases/pr/C96_atm3DVar.yaml index d45af6037a2..56605a37943 100644 --- a/dev/ci/cases/pr/C96_atm3DVar.yaml +++ b/dev/ci/cases/pr/C96_atm3DVar.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 idate: 2021122018 edate: 2021122106 nens: 0 diff --git a/dev/ci/cases/pr/C96_atm3DVar_extended.yaml b/dev/ci/cases/pr/C96_atm3DVar_extended.yaml index 6bf11397c93..0310dc58336 100644 --- a/dev/ci/cases/pr/C96_atm3DVar_extended.yaml +++ b/dev/ci/cases/pr/C96_atm3DVar_extended.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 idate: 2021122018 edate: 2021122118 nens: 0 diff --git a/dev/ci/cases/pr/C96_gcafs_cycled.yaml b/dev/ci/cases/pr/C96_gcafs_cycled.yaml index 619281898cd..54eb8a17671 100644 --- a/dev/ci/cases/pr/C96_gcafs_cycled.yaml +++ b/dev/ci/cases/pr/C96_gcafs_cycled.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 idate: 2021122012 edate: 2021122100 nens: 0 diff --git a/dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml b/dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml index f0d5dc39fe0..d27213bb25f 100644 --- a/dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml +++ b/dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 idate: 2021122012 edate: 2021122100 nens: 0 diff --git a/dev/ci/platforms/config.ursa b/dev/ci/platforms/config.ursa new file mode 100644 index 00000000000..cffec05c8b7 --- /dev/null +++ b/dev/ci/platforms/config.ursa @@ -0,0 +1,67 @@ +#!/usr/bin/bash + +######################################################################### +# config.ursa - Platform-specific configuration for Ursa +# +# This file contains environment variables used by CI/CD scripts, +# directories, and settings specific to the Ursa platform. +######################################################################### + +# Main CI root directory - Base directory for all CI/CD operations +export GFS_CI_ROOT=/scratch3/NCEPDEV/global/role.glopara/GFS_CI_CD + +# ICSDIR root directory - Contains initial condition data +# Used by create_experiment.py for setting up test cases +export ICSDIR_ROOT=/scratch3/NCEPDEV/global/role.glopara/data/ICSDIR + + +######################################################################### +# Jenkins configuration settings +######################################################################### + +# JENKINS launch directory for agent - Where Jenkins agents are launched from +export JENKINS_AGENT_LAUNCH_DIR=${GFS_CI_ROOT}/Jenkins/agent + +# JENKINS internal working directories for CI jobs (not for users use) +# Where Jenkins stores temporary files during CI job execution +export JENKINS_WORK_DIR=${GFS_CI_ROOT}/Jenkins/workspace + +# NOTE: JENKINS custom_workspace directory where CI jobs are run +# and is defined in $HOMEgfs/dev/ci/Jenkinsfile as custom_workspace +# /scratch3/NCEPDEV/global/role.glopara/CI + + +######################################################################### +# GitLab CI configuration +# These variables are referenced directly by launch_gitlab_runner.sh +######################################################################### + +# Used in the 'register' step of launch_gitlab_runner.sh for --url parameter +export GITLAB_URL=https://vlab.noaa.gov/gitlab-licensed +export GITLAB_RUNNER_NAME="RDHPCS Ursa" + +# Directory for GitLab builds +# Used in launch_gitlab_runner.sh for the --builds-dir parameter +# This works with --custom_build_dir-enabled=true to allow GIT_CLONE_PATH override +# from .gitlab-ci.yml to specify exact clone locations within this directory +export GITLAB_BUILDS_DIR=${GFS_CI_ROOT}/BUILDS/GITLAB + +# Directory for GitLab runner used by launch_gitlab_runner.sh +# This is where runner state/config files are stored (--working-directory parameter) +export GITLAB_RUNNER_DIR="${GFS_CI_ROOT}/GitLab/Runner" + +######################################################################### +# CTest functional test directories for pre stagged input data +######################################################################### +export STAGED_CTESTS=${GITLAB_BUILDS_DIR}/stable/RUNTESTS + + +######################################################################### +# CI CRON system configuration +######################################################################### +export GW_BASH_CI_ROOT=${GFS_CI_ROOT}/GFS_BASH_CI +export max_concurrent_cases=5 +export max_concurrent_pr=4 + +# HPC account which overides the default account +export HPC_ACCOUNT=fv3-cpu diff --git a/dev/ci/scripts/driver_weekly.sh b/dev/ci/scripts/driver_weekly.sh index b5b8201b5ba..6ec7172fee6 100755 --- a/dev/ci/scripts/driver_weekly.sh +++ b/dev/ci/scripts/driver_weekly.sh @@ -54,7 +54,7 @@ esac set +x source "${ROOT_DIR}/ush/module-setup.sh" module use "${ROOT_DIR}/modulefiles" -module load "module_gwsetup.${MACHINE_ID}" +module load "gw_setup.${MACHINE_ID}" set -x ######################################################### diff --git a/dev/ci/scripts/utils/ci_utils.sh b/dev/ci/scripts/utils/ci_utils.sh index 09660a4688f..a4bc8233a0f 100755 --- a/dev/ci/scripts/utils/ci_utils.sh +++ b/dev/ci/scripts/utils/ci_utils.sh @@ -126,7 +126,7 @@ function create_experiment () { local TAG="${2:-${pr_sha}}" cd "${HOMEgfs_}" || exit 1 case=$(basename "${yaml_config}" .yaml) || true - + echo "Using provided TAG: ${TAG} for pslot" export pslot=${case}_${TAG} @@ -201,12 +201,11 @@ function cleanup_experiment() { function build () { source "${HOMEgfs_}/dev/ci/platforms/config.${MACHINE_ID}" - # TODO: when it's safe to build on C6 compute nodes again, do so - if [[ "${MACHINE_ID}" == "gaeac6" ]]; then - "${HOMEgfs_}/sorc/build_all.sh" -k all - else - "${HOMEgfs_}/sorc/build_compute.sh" -A "${HPC_ACCOUNT}" all - fi + # TODO: when it is safe to build the GDASApp on the compute nodes again, do so (issues #3933 and #3932) + "${HOMEgfs_}/sorc/build_gdas.sh" & # Run in the background while the compute build is running + "${HOMEgfs_}/sorc/build_compute.sh" -A "${HPC_ACCOUNT}" gfs gcafs gefs sfs gsi + # Wait for the GDASApp to finish building + wait } diff --git a/dev/ci/scripts/utils/launch_java_agent.sh b/dev/ci/scripts/utils/launch_java_agent.sh index 830f6c7d11c..a8276a8402b 100755 --- a/dev/ci/scripts/utils/launch_java_agent.sh +++ b/dev/ci/scripts/utils/launch_java_agent.sh @@ -89,7 +89,7 @@ rm -f "${LOG}" HOMEgfs="${HOMEgfs_}" source "${HOMEgfs_}/ush/module-setup.sh" module use "${HOMEgfs_}/modulefiles" -module load "module_gwsetup.${MACHINE_ID}" +module load "gw_setup.${MACHINE_ID}" if [[ ${MACHINE_ID} == "noaacloud" ]]; then source "${HOMEgfs_}/dev/ci/platforms/config.${PW_CSP}" diff --git a/dev/jobs/upp.sh b/dev/jobs/upp.sh index 4d070e9c5c2..9e1d2f44bed 100755 --- a/dev/jobs/upp.sh +++ b/dev/jobs/upp.sh @@ -18,7 +18,7 @@ if [[ "${MACHINE_ID}" == "wcoss2" ]]; then set +x source "${HOMEgfs}/ush/module-setup.sh" module use "${HOMEgfs}/sorc/ufs_model.fd/FV3/upp/modulefiles" - module load "${MACHINE_ID}" + module load "${MACHINE_ID}_intel" module load prod_util module load cray-pals module load cfp @@ -44,7 +44,9 @@ if [[ "${MACHINE_ID}" == "wcoss2" ]]; then else source "${HOMEgfs}/dev/ush/load_fv3gfs_modules.sh" status=$? - if (( status != 0 )); then exit "${status}"; fi + if [[ ${status} -ne 0 ]]; then + exit "${status}"; + fi fi export job="upp" diff --git a/dev/parm/config/gcafs/config.resources b/dev/parm/config/gcafs/config.resources index 1d32c979a4e..1fee7c60684 100644 --- a/dev/parm/config/gcafs/config.resources +++ b/dev/parm/config/gcafs/config.resources @@ -47,6 +47,10 @@ case ${machine} in max_tasks_per_node=40 mem_node_max="96GB" ;; + "URSA") + max_tasks_per_node=192 + mem_node_max="360GB" + ;; "GAEAC5") max_tasks_per_node=128 mem_node_max="251GB" diff --git a/dev/parm/config/gefs/config.resources b/dev/parm/config/gefs/config.resources index bef6f4adf80..c16732ea4f6 100644 --- a/dev/parm/config/gefs/config.resources +++ b/dev/parm/config/gefs/config.resources @@ -19,6 +19,7 @@ echo "BEGIN: config.resources" case ${machine} in "WCOSS2") max_tasks_per_node=128;; "HERA") max_tasks_per_node=40;; + "URSA") max_tasks_per_node=192;; "ORION") max_tasks_per_node=40;; "HERCULES") max_tasks_per_node=80;; "GAEAC5") max_tasks_per_node=128;; diff --git a/dev/parm/config/gfs/config.resources b/dev/parm/config/gfs/config.resources index 5a2714dce73..e545cc471c8 100644 --- a/dev/parm/config/gfs/config.resources +++ b/dev/parm/config/gfs/config.resources @@ -45,6 +45,10 @@ case ${machine} in max_tasks_per_node=40 mem_node_max="96GB" ;; + "URSA") + max_tasks_per_node=192 + mem_node_max="360GB" + ;; "GAEAC5") max_tasks_per_node=128 mem_node_max="251GB" @@ -1036,7 +1040,7 @@ case ${step} in ntasks=1 threads_per_task=1 tasks_per_node=1 - memory="5G" + memory="10G" ;; "vminmon") diff --git a/dev/parm/config/gfs/config.resources.URSA b/dev/parm/config/gfs/config.resources.URSA new file mode 100644 index 00000000000..a00edf5e892 --- /dev/null +++ b/dev/parm/config/gfs/config.resources.URSA @@ -0,0 +1,86 @@ +#! /usr/bin/env bash + +# Hera-specific job resources + +case ${step} in + "fcst" | "efcs") + case "${CASE}" in + "C768") + export tasks_per_node=144 + ;; + "C1152") + export tasks_per_node=144 + ;; + *) + # Nothing to do for other resolutions + true + ;; + esac + ;; + "anal") + case ${CASE} in + "C768") + export threads_per_task=8 + ;; + "C1152") + export threads_per_task=16 + ;; + *) + ;; + esac + if [[ "${CASE}" == "C768" || "${CASE}" == "C1152" ]]; then + # Make ntasks a multiple of 16 + if [[ "${RUN}" == "gdas" ]]; then + export ntasks=784 + elif [[ "${RUN}" == "gfs" ]]; then + export ntasks=832 + fi + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + fi + ;; + + "eupd") + case "${CASE}" in + "C768") + export ntasks=80 + export threads_per_task=20 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + ;; + "C1152") + export ntasks=480 + export threads_per_task=6 + export tasks_per_node=8 + ;; + *) + ;; + esac + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + ;; + + "aeroanlvar") + export tasks_per_node=48 + ;; + + "marinebmat") + case ${OCNRES} in + "025") + tasks_per_node=$(( max_tasks_per_node / 4 )) + ;; + *) + ;; + esac + ;; + + "ocnanalecen") + case ${OCNRES} in + "025") + ntasks=288 + tasks_per_node=12 + ;; + *) + ;; + esac + ;; + *) + ;; +esac diff --git a/dev/parm/config/sfs/config.resources b/dev/parm/config/sfs/config.resources index 6fbe27363cb..803e6282711 100644 --- a/dev/parm/config/sfs/config.resources +++ b/dev/parm/config/sfs/config.resources @@ -19,6 +19,7 @@ echo "BEGIN: config.resources" case ${machine} in "WCOSS2") max_tasks_per_node=128;; "HERA") max_tasks_per_node=40;; + "URSA") max_tasks_per_node=192;; "ORION") max_tasks_per_node=40;; "HERCULES") max_tasks_per_node=80;; "GAEAC6") max_tasks_per_node=192;; diff --git a/dev/ush/gw_setup.sh b/dev/ush/gw_setup.sh index a101ea8b79d..e096d508aff 100755 --- a/dev/ush/gw_setup.sh +++ b/dev/ush/gw_setup.sh @@ -8,12 +8,19 @@ # This script should be SOURCED to properly setup the environment. # -HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +# Determine if HOMEgfs is already set +unset_homegfs=NO +if [[ -z "${HOMEgfs+x}" ]]; then + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" + HOMEgfs=$(cd "${script_dir}" && git rev-parse --show-toplevel) + export HOMEgfs + unset_homegfs=YES +fi source "${HOMEgfs}/ush/detect_machine.sh" source "${HOMEgfs}/ush/module-setup.sh" module use "${HOMEgfs}/modulefiles" -module load "module_gwsetup.${MACHINE_ID}" +module load "gw_setup.${MACHINE_ID}" err=$? if [[ "${err}" -ne 0 ]]; then echo "FATAL ERROR: Failed to load module_gwsetup.${MACHINE_ID}" @@ -25,3 +32,8 @@ if [[ -d "${HOMEgfs}/sorc/wxflow/src" ]]; then PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/sorc/wxflow/src" export PYTHONPATH fi + +if [[ ${unset_homegfs} == "YES" ]]; then + unset HOMEgfs +fi + diff --git a/dev/ush/load_fv3gfs_modules.sh b/dev/ush/load_fv3gfs_modules.sh deleted file mode 100755 index 5fe5e0449e9..00000000000 --- a/dev/ush/load_fv3gfs_modules.sh +++ /dev/null @@ -1,63 +0,0 @@ -#! /usr/bin/env bash - -############################################################### -if [[ "$-" == *x* ]]; then - set_x=YES -else - set_x=NO -fi - -if [[ "${DEBUG_WORKFLOW:-NO}" == "NO" ]]; then - echo "Loading modules quietly..." - set +x -fi - -# Setup runtime environment by loading modules -ulimit_s=$(ulimit -S -s) - -# Find module command and purge: -source "${HOMEgfs}/ush/detect_machine.sh" -source "${HOMEgfs}/ush/module-setup.sh" - -# Source versions file for runtime -source "${HOMEgfs}/versions/run.ver" - -# Load our modules: -module use "${HOMEgfs}/modulefiles" - -case "${MACHINE_ID}" in -"wcoss2" | "hera" | "orion" | "hercules" | "gaeac5" | "gaeac6" | "noaacloud") - module load "module_base.${MACHINE_ID}" - export err=$? - if [[ ${err} -ne 0 ]]; then - echo "FATAL ERROR: Failed to load module_base.${MACHINE_ID}" - exit 1 - fi - ;; -*) - echo "WARNING: UNKNOWN PLATFORM" - ;; -esac - -module list - -# If this function exists in the environment, run it; else set -x if it was set on entering this script -ftype=$(type -t set_trace || echo "") -if [[ "${ftype}" == "function" ]]; then - set_trace -elif [[ "${set_x}" == "YES" ]]; then - set -x -fi - -# Set up the PYTHONPATH to include wxflow from HOMEgfs -if [[ -d "${HOMEgfs}/sorc/wxflow/src" ]]; then - PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/sorc/wxflow/src" -fi - -# Add HOMEgfs/ush/python to PYTHONPATH -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python" -export PYTHONPATH - -# Restore stack soft limit: -ulimit -S -s "${ulimit_s}" -unset ulimit_s diff --git a/dev/ush/load_gw_run_modules.sh b/dev/ush/load_gw_run_modules.sh new file mode 100755 index 00000000000..660463ed0c2 --- /dev/null +++ b/dev/ush/load_gw_run_modules.sh @@ -0,0 +1,99 @@ +#! /usr/bin/env bash + +############################################################### +if [[ "$-" == *x* ]]; then + set_x=YES +else + set_x=NO +fi + +if [[ "${DEBUG_WORKFLOW:-NO}" == "NO" ]]; then + echo "Loading modules quietly..." + set +x +fi + +# Setup runtime environment by loading modules +ulimit_s=$( ulimit -S -s ) + +# Test if HOMEgfs is defined. If not, then try to determine it with git rev-parse +_unset_homegfs="NO" +if [[ -z ${HOMEgfs+x} ]]; then + echo "INFO HOMEgfs is not defined. Attempting to find the global-workflow root directory" + # HOMEgfs will be removed from the environment at the end of this script + _unset_homegfs="YES" + + script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + HOMEgfs=$(cd "${script_dir}" && git rev-parse --show-toplevel) + export HOMEgfs + err=$? + if [[ ${err} -ne 0 ]]; then + is_git_dir=$( cd -- "${script_dir}" &> /dev/null && git rev-parse --is-inside-work-tree) + git_stat=$? + if [[ ${git_stat} -ne 0 || ${is_git_dir} != "true" ]]; then + echo "FATAL ERROR unable to determine the root because it is not a git repository." + else + echo "FATAL ERROR unable to determine the root because git rev-parse --show-toplevel failed for an unknown reason" + fi + echo " Unable to load modules. Exiting" + exit 1 + fi +fi + +# Find module command and purge: +source "${HOMEgfs}/ush/detect_machine.sh" +source "${HOMEgfs}/ush/module-setup.sh" + +# Source versions file for runtime +if [[ -f "${HOMEgfs}/versions/run.ver" ]]; then + source "${HOMEgfs}/versions/run.ver" +else + echo "FATAL ERROR ${HOMEgfs}/versions/run.ver does not exist!" + echo "HINT: Run link_workflow.sh first." + exit 1 +fi + +# Load our modules: +module use "${HOMEgfs}/modulefiles" + +case "${MACHINE_ID}" in +"wcoss2" | "ursa" | "hera" | "orion" | "hercules" | "gaeac5" | "gaeac6" | "noaacloud") + target_module="gw_run.${MACHINE_ID}" + module load "${target_module}" + export err=$? + if [[ ${err} -ne 0 ]]; then + echo "FATAL ERROR: Failed to load ${target_module}" + exit 1 + fi + ;; +*) + echo "WARNING: UNKNOWN PLATFORM" + ;; +esac + +module list + +# If this function exists in the environment, run it; else set -x if it was set on entering this script +ftype=$(type -t set_trace || echo "") +if [[ "${ftype}" == "function" ]]; then + set_trace +elif [[ "${set_x}" == "YES" ]]; then + set -x +fi + +# Set up the PYTHONPATH to include wxflow from HOMEgfs +if [[ -d "${HOMEgfs}/sorc/wxflow/src" ]]; then + PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/sorc/wxflow/src" +fi + +# Add HOMEgfs/ush/python to PYTHONPATH +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python" +export PYTHONPATH + +# Restore stack soft limit: +ulimit -S -s "${ulimit_s}" +unset ulimit_s + +# Unset HOMEgfs if it was not set at the beginning of this script +if [[ ${_unset_homegfs} == "YES" ]]; then + unset HOMEgfs +fi diff --git a/dev/ush/load_ufsda_modules.sh b/dev/ush/load_ufsda_modules.sh index 7d6e2379bc7..e44577328af 100755 --- a/dev/ush/load_ufsda_modules.sh +++ b/dev/ush/load_ufsda_modules.sh @@ -40,31 +40,31 @@ source "${HOMEgfs}/ush/module-setup.sh" module use "${HOMEgfs}/sorc/gdas.cd/modulefiles" case "${MACHINE_ID}" in -"hera" | "orion" | "hercules" | "wcoss2" | "gaeac5" | "gaeac6") - #TODO: Remove LMOD_TMOD_FIND_FIRST line when spack-stack on WCOSS2 - if [[ "${MACHINE_ID}" == "wcoss2" ]]; then - export LMOD_TMOD_FIND_FIRST=yes - # TODO: Add path to GDASApp libraries and cray-mpich as temporary patches - # TODO: Remove LD_LIBRARY_PATH lines as soon as permanent solutions are available - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgfs}/sorc/gdas.cd/build/lib" - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/cray/pe/mpich/8.1.19/ofi/intel/19.0/lib" - fi - module load "${MODS}/${MACHINE_ID}" - export err=$? - if [[ ${err} -ne 0 ]]; then - echo "FATAL ERROR: Failed to load ${MODS}/${MACHINE_ID}" - exit 1 - fi - ncdump=$(command -v ncdump) - NETCDF=$(echo "${ncdump}" | cut -d " " -f 3) - export NETCDF - ;; -"acorn") - echo WARNING: UFSDA NOT SUPPORTED ON THIS PLATFORM - ;; -*) - echo "WARNING: UNKNOWN PLATFORM" - ;; + ("hera" | "orion" | "hercules" | "wcoss2" | "gaeac5" | "gaeac6" | "ursa") + #TODO: Remove LMOD_TMOD_FIND_FIRST line when spack-stack on WCOSS2 + if [[ "${MACHINE_ID}" == "wcoss2" ]]; then + export LMOD_TMOD_FIND_FIRST=yes + # TODO: Add path to GDASApp libraries and cray-mpich as temporary patches + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgfs}/sorc/gdas.cd/build/lib" + # TODO: Remove LD_LIBRARY_PATH line as soon as permanent solution is available + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/cray/pe/mpich/8.1.29/ofi/intel/2022.1/lib" + fi + module load "${MODS}/${MACHINE_ID}" + export err=$? + if [[ ${err} -ne 0 ]]; then + echo "FATAL ERROR: Failed to load ${MODS}/${MACHINE_ID}" + exit 1 + fi + ncdump=$( command -v ncdump ) + NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) + export NETCDF + ;; + ("acorn") + echo WARNING: UFSDA NOT SUPPORTED ON THIS PLATFORM + ;; + *) + echo "WARNING: UNKNOWN PLATFORM" + ;; esac module list diff --git a/dev/ush/load_ufswm_modules.sh b/dev/ush/load_ufswm_modules.sh index 02217d5d41f..c037f54b9a2 100755 --- a/dev/ush/load_ufswm_modules.sh +++ b/dev/ush/load_ufswm_modules.sh @@ -27,10 +27,12 @@ if [[ "${MACHINE_ID}" = "wcoss2" ]]; then module load craype-network-ucx module load cray-mpich-ucx module load python/3.8.6 + module load wgrib2 else export UTILROOT=${prod_util_ROOT} + source "${HOMEgfs}/versions/run.ver" + module load "wgrib2/${wgrib2_ver}" fi -module load wgrib2 export WGRIB2=wgrib2 module list diff --git a/dev/workflow/hosts.py b/dev/workflow/hosts.py index 8eef5808d23..1487cbf367c 100644 --- a/dev/workflow/hosts.py +++ b/dev/workflow/hosts.py @@ -15,7 +15,7 @@ class Host: Gather Host specific information. """ - SUPPORTED_HOSTS = ['HERA', 'ORION', 'HERCULES', 'WCOSS2', 'CONTAINER', + SUPPORTED_HOSTS = ['HERA', 'URSA', 'ORION', 'HERCULES', 'WCOSS2', 'CONTAINER', 'GAEAC5', 'GAEAC6', 'AWSPW', 'AZUREPW', 'GOOGLEPW'] def __init__(self, host=None): @@ -55,8 +55,31 @@ def detect(self) -> None: # Detect the machine since MACHINE_ID is not set if os.path.exists('/scratch3/NCEPDEV'): - self.machine = 'HERA' + # Hera or Ursa + self.machine = "" + + # Open the mountinfo file and check if /home is mounted to "home_ursa" or "home_hera" + # NOTE: the github runners do not have a /home directory, so self.machine will be unset + with open('/proc/self/mountinfo') as f: + for line in f: + fields = line.strip().split() + mount_point = fields[4] + if mount_point == "/home": + mount_source = fields[9] + if "hera" in mount_source.lower(): + self.machine = "HERA" + elif "ursa" in mount_source.lower(): + self.machine = "URSA" + + # TODO: When Hera is no longer used, remove this check and switch to Ursa. + # Check if this is the GitHub runner + if self.machine != 'HERA' and self.machine != 'URSA': + machine = socket.gethostname().upper() + print(f'Detected host {machine}; assuming this is a GitHub runner.') + self.machine = 'HERA' + elif os.path.exists('/work/noaa'): + # Orion or Hercules self.machine = socket.gethostname().split("-", 1)[0].upper() elif os.path.exists('/lfs/f1'): self.machine = 'WCOSS2' diff --git a/dev/workflow/hosts/hera.yaml b/dev/workflow/hosts/hera.yaml index c6b0d15179e..cd964557a92 100644 --- a/dev/workflow/hosts/hera.yaml +++ b/dev/workflow/hosts/hera.yaml @@ -1,18 +1,18 @@ # Paths DMPDIR: '/scratch3/NCEPDEV/global/role.glopara/dump' -BASE_GIT: '/scratch1/NCEPDEV/global/glopara/git' +BASE_GIT: '/scratch3/NCEPDEV/global/role.glopara/git_hera' BASE_DATA: '/scratch3/NCEPDEV/global/role.glopara/data' BASE_IC: '/scratch3/NCEPDEV/global/role.glopara/data/ICSDIR' -AERO_INPUTS_DIR: '/scratch1/NCEPDEV/global/glopara/data/GEFS_ExtData/20250310' +AERO_INPUTS_DIR: '/scratch3/NCEPDEV/global/role.glopara/data/GEFS_ExtData/20250310' PACKAGEROOT: '/scratch3/NCEPDEV/global/role.glopara/nwpara' HOMEDIR: '/scratch3/NCEPDEV/global/${USER}' STMP: '/scratch3/NCEPDEV/stmp/${USER}' PTMP: '/scratch3/NCEPDEV/stmp/${USER}' NOSCRUB: '${HOMEDIR}' -COMINsyn: '/scratch1/NCEPDEV/global/glopara/com/gfs/prod/syndat' -COMINecmwf: '/scratch1/NCEPDEV/global/glopara/data/external_gempak/ecmwf' -COMINnam: '/scratch1/NCEPDEV/global/glopara/data/external_gempak/nam' -COMINukmet: '/scratch1/NCEPDEV/global/glopara/data/external_gempak/ukmet' +COMINsyn: '/scratch3/NCEPDEV/global/role.glopara/com/gfs/prod/syndat' +COMINecmwf: '/scratch3/NCEPDEV/global/role.glopara/data/external_gempak/ecmwf' +COMINnam: '/scratch3/NCEPDEV/global/role.glopara/data/external_gempak/nam' +COMINukmet: '/scratch3/NCEPDEV/global/role.glopara/data/external_gempak/ukmet' # BQS properties SCHEDULER: slurm QUEUE: batch diff --git a/dev/workflow/hosts/ursa.yaml b/dev/workflow/hosts/ursa.yaml new file mode 100644 index 00000000000..bc54d690160 --- /dev/null +++ b/dev/workflow/hosts/ursa.yaml @@ -0,0 +1,32 @@ +# Paths +DMPDIR: '/scratch3/NCEPDEV/global/role.glopara/dump' +BASE_GIT: '/scratch3/NCEPDEV/global/role.glopara/git' +BASE_DATA: '/scratch3/NCEPDEV/global/role.glopara/data' +BASE_IC: '/scratch3/NCEPDEV/global/role.glopara/data/ICSDIR' +AERO_INPUTS_DIR: '/scratch3/NCEPDEV/global/role.glopara/data/GEFS_ExtData/20250310' +PACKAGEROOT: '/scratch3/NCEPDEV/global/role.glopara/nwpara' +HOMEDIR: '/scratch3/NCEPDEV/global/${USER}' +STMP: '/scratch3/NCEPDEV/stmp/${USER}' +PTMP: '/scratch3/NCEPDEV/stmp/${USER}' +NOSCRUB: '${HOMEDIR}' +COMINsyn: '/scratch3/NCEPDEV/global/role.glopara/com/gfs/prod/syndat' +COMINecmwf: '/scratch3/NCEPDEV/global/role.glopara/data/external_gempak/ecmwf' +COMINnam: '/scratch3/NCEPDEV/global/role.glopara/data/external_gempak/nam' +COMINukmet: '/scratch3/NCEPDEV/global/role.glopara/data/external_gempak/ukmet' +# BQS properties +SCHEDULER: slurm +QUEUE: batch +PARTITION_BATCH: u1-compute +PARTITION_SERVICE: u1-service +# HPSS properties +HPSS_PROJECT: emc-global +ARCHCOM_TO: 'hpss' +ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' +CHGRP_RSTPROD: 'YES' +CHGRP_CMD: 'chgrp rstprod' +# Features +SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] +DO_ARCHCOM: 'YES' +DO_AWIPS: 'NO' +MAKE_NSSTBUFR: 'NO' +MAKE_ACFTBUFR: 'NO' diff --git a/dev/workflow/rocoto/tasks.py b/dev/workflow/rocoto/tasks.py index 37dc943e7aa..e6bf446d98b 100644 --- a/dev/workflow/rocoto/tasks.py +++ b/dev/workflow/rocoto/tasks.py @@ -11,11 +11,11 @@ class Tasks: - SERVICE_TASKS = ['arch_vrfy', 'earc_vrfy', 'stage_ic', 'cleanup', 'globus', 'ens_globus'] + SERVICE_TASKS = ['arch_vrfy', 'earc_vrfy', 'stage_ic', 'globus', 'ens_globus'] DTN_TASKS = ['arch_tars', 'earc_tars', 'fetch'] VALID_TASKS = ['aerosol_init', 'stage_ic', 'gen_control_ic', 'fetch', 'globus', 'ens_globus', 'prep_sfc', 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch_vrfy', 'arch_tars', 'cleanup', - 'ecen_fv3jedi', 'analcalc_fv3jedi', + 'ecen_fv3jedi', 'analcalc_fv3jedi', 'cleanup', 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', 'prep_emissions', 'prepoceanobs', 'marineanlinit', 'marineanlletkf', 'marinebmatinit', 'marinebmat', 'marineanlvar', diff --git a/env/AWSPW.env b/env/AWSPW.env index 68fe7a12ff6..8080dd0f76c 100755 --- a/env/AWSPW.env +++ b/env/AWSPW.env @@ -85,6 +85,12 @@ elif [[ "${step}" = "atmos_products" ]]; then elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then + if [[ "${step}" == "anal" ]]; then + # This setting is required for the GSI due to an IMPI bug (see https://github.com/NOAA-EMC/GSI/issues/887 for details) + # TODO: Remove this when Intel has a fix for this bug in a future release + export I_MPI_ADJUST_ALLREDUCE=5 + fi + export CFP_MP="YES" export USE_CFP="YES" export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" diff --git a/env/HERA.env b/env/HERA.env index c1e3edd196a..67c74966d16 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -15,7 +15,6 @@ export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" #export POSTAMBLE_CMD='report-mem' # Configure MPI environment -#export I_MPI_ADJUST_ALLREDUCE=5 #export MPI_BUFS_PER_PROC=2048 #export MPI_BUFS_PER_HOST=2048 #export MPI_GROUP_MAX=256 @@ -176,6 +175,12 @@ elif [[ "${step}" = "analcalc_fv3jedi" ]]; then elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then + if [[ "${step}" == "anal" ]]; then + # This setting is required for the GSI due to an IMPI bug (see https://github.com/NOAA-EMC/GSI/issues/887 for details) + # TODO: Remove this when Intel has a fix for this bug in a future release + export I_MPI_ADJUST_ALLREDUCE=5 + fi + export MKL_NUM_THREADS=4 export MKL_CBWR=AUTO diff --git a/env/HERCULES.env b/env/HERCULES.env index bbbfb6e6a52..ab110b7de2d 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -177,6 +177,12 @@ case ${step} in ;; "anal" | "analcalc") + if [[ "${step}" == "anal" ]]; then + # This setting is required for the GSI due to an IMPI bug (see https://github.com/NOAA-EMC/GSI/issues/887 for details) + # TODO: Remove this when Intel has a fix for this bug in a future release + export I_MPI_ADJUST_ALLREDUCE=5 + fi + export MKL_NUM_THREADS=4 export MKL_CBWR=AUTO diff --git a/env/ORION.env b/env/ORION.env index 1cc6dde1777..3fcd6bb6bcd 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -177,6 +177,12 @@ elif [[ "${step}" = "analcalc_fv3jedi" ]]; then elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then + if [[ "${step}" == "anal" ]]; then + # This setting is required for the GSI due to an IMPI bug (see https://github.com/NOAA-EMC/GSI/issues/887 for details) + # TODO: Remove this when Intel has a fix for this bug in a future release + export I_MPI_ADJUST_ALLREDUCE=5 + fi + export MKL_NUM_THREADS=4 export MKL_CBWR=AUTO diff --git a/env/URSA.env b/env/URSA.env new file mode 100644 index 00000000000..255b4fcfa55 --- /dev/null +++ b/env/URSA.env @@ -0,0 +1,326 @@ +#! /usr/bin/env bash + +if [[ $# -ne 1 ]]; then + + echo "Must specify an input argument to set runtime environment variables!" + exit 1 + +fi + +step=$1 + +export launcher="srun -l --export=ALL --hint=nomultithread" +export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" + +#export POSTAMBLE_CMD='report-mem' + +# Configure MPI environment +export OMP_STACKSIZE=2048000 +export NTHSTACK=1024000000 + +# Setting stacksize to unlimited on login nodes is prohibited +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + ulimit -s unlimited + ulimit -a +fi + +# Calculate common variables +# Check first if the dependent variables are set +if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then + max_threads_per_task=$((max_tasks_per_node / tasks_per_node)) + NTHREADSmax=${threads_per_task:-${max_threads_per_task}} + NTHREADS1=${threads_per_task:-1} + if [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]]; then + NTHREADSmax=${max_threads_per_task} + fi + if [[ ${NTHREADS1} -gt ${max_threads_per_task} ]]; then + NTHREADS1=${max_threads_per_task} + fi + APRUN_default="${launcher} -n ${ntasks}" +else + echo "ERROR config.resources must be sourced before sourcing URSA.env" + exit 2 +fi + +if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then + + export POE="NO" + export BACK="NO" + export sys_tp="URSA" + export launcher_PREP="srun --hint=nomultithread" + +elif [[ "${step}" = "prep_emissions" ]]; then + + export APRUN="${APRUN_default}" + +elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then + + export USE_CFP="YES" + if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi + export wavempexec=${launcher} + export wave_mpmd=${mpmd_opt} + +elif [[ "${step}" = "atmanlvar" ]]; then + + export NTHREADS_ATMANLVAR=${NTHREADSmax} + export APRUN_ATMANLVAR="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLVAR}" + +elif [[ "${step}" = "atmensanlobs" ]]; then + + export NTHREADS_ATMENSANLOBS=${NTHREADSmax} + export APRUN_ATMENSANLOBS="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLOBS}" + +elif [[ "${step}" = "atmensanlsol" ]]; then + + export NTHREADS_ATMENSANLSOL=${NTHREADSmax} + export APRUN_ATMENSANLSOL="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLSOL}" + +elif [[ "${step}" = "atmensanlletkf" ]]; then + + export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} + export APRUN_ATMENSANLLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}" + +elif [[ "${step}" = "atmensanlfv3inc" ]]; then + + export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} + export APRUN_ATMENSANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" + +elif [[ "${step}" = "aeroanlvar" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + export NTHREADS_AEROANL=${NTHREADSmax} + export APRUN_AEROANL="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANL}" + +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANLGENB}" + +elif [[ "${step}" = "atmanlfv3inc" ]]; then + + export NTHREADS_ATMANLFV3INC=${NTHREADSmax} + export APRUN_ATMANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" + +elif [[ "${step}" = "anlstat" ]]; then + + export NTHREADS_ANLSTAT=${NTHREADSmax} + export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}" + +elif [[ "${step}" = "prepobsaero" ]]; then + + export NTHREADS_PREPOBSAERO=${NTHREADS1} + export APRUN_PREPOBSAERO="${APRUN_default} --cpus-per-task=${NTHREADS_PREPOBSAERO}" + +elif [[ "${step}" = "snowanl" ]]; then + + export APRUN_CALCFIMS="${launcher} -n 1" + + export NTHREADS_SNOWANL=${NTHREADSmax} + export APRUN_SNOWANL="${APRUN_default} --mem=0 --cpus-per-task=${NTHREADS_SNOWANL}" + + export APRUN_APPLY_INCR="${launcher} -n 6" + +elif [[ "${step}" = "esnowanl" ]]; then + + export APRUN_CALCFIMS="${launcher} -n 1" + + export NTHREADS_ESNOWANL=${NTHREADSmax} + export APRUN_ESNOWANL="${APRUN_default} --mem=0 --cpus-per-task=${NTHREADS_ESNOWANL}" + + export APRUN_APPLY_INCR="${launcher} -n 6" + +elif [[ "${step}" = "marinebmat" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUN_MARINEBMAT="${APRUN_default}" + +elif [[ "${step}" = "marineanlvar" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUN_MARINEANLVAR="${APRUN_default}" + +elif [[ "${step}" = "marineanlchkpt" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUN_MARINEANLCHKPT="${APRUN_default}" + + export APRUN_MARINEANLOBSSTATS="${launcher} -n 1" + +elif [[ "${step}" = "ocnanalecen" ]]; then + + export NTHREADS_OCNANALECEN=${NTHREADSmax} + export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" + +elif [[ "${step}" = "marineanlletkf" ]]; then + + export NTHREADS_MARINEANLLETKF=${NTHREADSmax} + export APRUN_MARINEANLLETKF=${APRUN_default} + +elif [[ "${step}" = "ecen_fv3jedi" ]]; then + + export NTHREADS_ECEN_FV3JEDI=${NTHREADSmax} + export APRUN_CORRECTION_INCREMENT="${launcher} -n ${ntasks_correction_increment} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}" + export APRUN_ENSEMBLE_RECENTER="${launcher} -n ${ntasks_ensemble_recenter} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}" + +elif [[ "${step}" = "analcalc_fv3jedi" ]]; then + + export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax} + export APRUN_ANALCALC_FV3JEDI="${APRUN_default} --cpus-per-task=${NTHREADS_ANALCALC_FV3JEDI}" + +elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then + + export MKL_NUM_THREADS=4 + export MKL_CBWR=AUTO + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + export NTHREADS_GSI=${NTHREADSmax} + export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}" + + export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} + if [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]]; then + export NTHREADS_CALCINC=${max_threads_per_task} + fi + export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}" + + export NTHREADS_CYCLE=${threads_per_task_cycle:-12} + if [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]]; then + export NTHREADS_CYCLE=${max_tasks_per_node} + fi + ntasks_cycle=${ntiles:-6} + export APRUN_CYCLE="${launcher} -n ${ntasks_cycle} --cpus-per-task=${NTHREADS_CYCLE}" + + export NTHREADS_GAUSFCANL=1 + ntasks_gausfcanl=${ntasks_gausfcanl:-1} + export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}" + +elif [[ "${step}" = "offlineanl" ]]; then + + export NTHREADS_CHGRES=${threads_per_task_chgres:-12} + if [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]]; then + export NTHREADS_CHGRES=${max_tasks_per_node} + fi + export APRUN_CHGRES="time" + + export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} + if [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]]; then + export NTHREADS_CALCINC=${max_threads_per_task} + fi + export APRUN_CALCINC="${APRUN_default} --cpus-per-task=${NTHREADS_CALCINC}" + +elif [[ "${step}" = "sfcanl" ]]; then + + export NTHREADS_CYCLE=${threads_per_task:-14} + export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" + + # REGRID requires 6 tasks for reproducibility + ntasks_regrid=6 + export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + +elif [[ "${step}" = "eobs" ]]; then + + export MKL_NUM_THREADS=4 + export MKL_CBWR=AUTO + + export NTHREADS_GSI=${NTHREADSmax} + export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}" + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + +elif [[ "${step}" = "eupd" ]]; then + + export NTHREADS_ENKF=${NTHREADSmax} + export APRUN_ENKF="${launcher} -n ${ntasks_enkf:-${ntasks}} --cpus-per-task=${NTHREADS_ENKF}" + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + +elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then + + (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) + (( ufs_ntasks = nnodes*tasks_per_node )) + # With ESMF threading, the model wants to use the full node + export APRUN_UFS="${launcher} -n ${ufs_ntasks}" + unset nnodes ufs_ntasks + +elif [[ "${step}" = "upp" ]]; then + + export NTHREADS_UPP=${NTHREADS1} + export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}" + +elif [[ "${step}" = "atmos_products" ]]; then + + export USE_CFP="YES" # Use MPMD for downstream product generation on Ursa + +elif [[ "${step}" = "oceanice_products" ]]; then + + export NTHREADS_OCNICEPOST=${NTHREADS1} + export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}" + +elif [[ "${step}" = "ecen" ]]; then + + export NTHREADS_ECEN=${NTHREADSmax} + export APRUN_ECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ECEN}" + + export NTHREADS_CHGRES=${threads_per_task_chgres:-12} + if [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]]; then + export NTHREADS_CHGRES=${max_tasks_per_node} + fi + export APRUN_CHGRES="time" + + export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} + if [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]]; then + export NTHREADS_CALCINC=${max_threads_per_task} + fi + export APRUN_CALCINC="${APRUN_default} --cpus-per-task=${NTHREADS_CALCINC}" + +elif [[ "${step}" = "esfc" ]]; then + + export NTHREADS_ESFC=${threads_per_task_esfc:-${max_threads_per_task}} + export APRUN_ESFC="${APRUN_default} --cpus-per-task=${NTHREADS_ESFC}" + + export NTHREADS_CYCLE=${threads_per_task_cycle:-14} + if [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]]; then + export NTHREADS_CYCLE=${max_tasks_per_node} + fi + export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" + + # REGRID requires 6 tasks for reproducibility + ntasks_regrid=6 + export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + +elif [[ "${step}" = "epos" ]]; then + + export NTHREADS_EPOS=${NTHREADSmax} + export APRUN_EPOS="${APRUN_default} --cpus-per-task=${NTHREADS_EPOS}" + +elif [[ "${step}" = "postsnd" ]]; then + + export CFP_MP="YES" + + export NTHREADS_POSTSND=${NTHREADS1} + export APRUN_POSTSND="${APRUN_default} --cpus-per-task=${NTHREADS_POSTSND}" + + export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1} + if [[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]]; then + export NTHREADS_POSTSNDCFP=${max_threads_per_task} + fi + export APRUN_POSTSNDCFP="${launcher} -n ${ntasks_postsndcfp} ${mpmd_opt}" + +elif [[ "${step}" = "awips" ]]; then + + export NTHREADS_AWIPS=${NTHREADS1} + export APRUN_AWIPSCFP="${APRUN_default} ${mpmd_opt}" + +elif [[ "${step}" = "fit2obs" ]]; then + + export NTHREADS_FIT2OBS=${NTHREADS1} + export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}" + +fi diff --git a/modulefiles/gw_run.common.lua b/modulefiles/gw_run.common.lua new file mode 100644 index 00000000000..997466b5748 --- /dev/null +++ b/modulefiles/gw_run.common.lua @@ -0,0 +1,64 @@ +help([[ +Load common modules for all systems +]]) + +local stack_compiler=(os.getenv("stack_compiler") or "None") +local stack_compiler_ver=(os.getenv("stack_compiler_ver") or "None") +local stack_mpi=(os.getenv("stack_mpi") or "None") +local stack_mpi_ver=(os.getenv("stack_mpi_ver") or "None") +local stack_core_path=(os.getenv("stack_core_path") or "None") +if stack_compiler == "None" or stack_compiler_ver == "None" or + stack_mpi == "None" or stack_mpi_ver == "None" or + stack_core_path == "None" then + LmodError("FATAL ERROR One or more spack-stack variables is undefined!") +end + +prepend_path("MODULEPATH", stack_core_path) + +local stack_compiler_module=pathJoin("stack-" .. stack_compiler, stack_compiler_ver) +local stack_mpi_module=pathJoin("stack-" .. stack_mpi, stack_mpi_ver) + +load(stack_compiler_module) +load(stack_mpi_module) + +local common_modules = { + "stack-python", + "cmake", + "jasper", + "libpng", + "cdo", + "hdf5", + "netcdf-c", + "netcdf-fortran", + "esmf", + "nco", + "prod_util", + "grib-util", + "g2tmpl", + "gsi-ncdiag", + "crtm", + "bufr", + --"wgrib2", temporarily disable wgrib2 until it is installed with ipolates + "py-f90nml", + "py-netcdf4", + "py-pyyaml", + "py-jinja2", + "py-pandas", + "py-python-dateutil", + "py-xarray", + -- TODO: Reenable when MET/METplus and verif-global are at compatible versions + -- "met", + -- "metplus", +} + +for _, name in pairs(common_modules) do + -- Version variables have "_"s where the module name has "-"s + local ver_name = string.gsub(name, "-", "_") .. "_ver" + ver = os.getenv(ver_name) or "None" + if ver == "None" then + LmodError("FATAL ERROR version variable " .. ver_name .. " is not set! Unable to load module!") + end + load(pathJoin(name, ver)) +end + +setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) diff --git a/modulefiles/gw_run.gaeac6.lua b/modulefiles/gw_run.gaeac6.lua new file mode 100644 index 00000000000..b509ba83997 --- /dev/null +++ b/modulefiles/gw_run.gaeac6.lua @@ -0,0 +1,33 @@ +help([[ +Load environment to run GFS on Gaea C6 +]]) + +-- Test that HOMEgfs is set. +-- If not, load_gw_run_modules.sh was not sourced to load this module. +local homegfssdir=os.getenv("HOMEgfs") or "None" +if (homegfssdir == "None") then + LmodError("FATAL ERROR HOMEgfs variable is unset.\n" .. + "Please \"source ush/load_gw_run_modules.sh\" rather than loading this module directly.\n") +end + +load("gw_run.common") + +load(pathJoin("perlbrew", (os.getenv("perl_ver") or "None"))) + +prepend_path("MODULEPATH", pathJoin("/gpfs/f6/drsa-precip3/world-shared/role.glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) + +prepend_path("MODULEPATH", pathJoin("/gpfs/f6/drsa-precip3/world-shared/role.glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) + +local hsi_mod_path=(os.getenv("hsi_mod_path") or "None") +append_path("MODULEPATH", hsi_mod_path) +load(pathJoin("hsi", (os.getenv("hsi_ver") or "None"))) + +-- Point to the wgrib2 executable with ipolates +setenv("WGRIB2", "/autofs/ncrc-svm1_proj/epic/c6/spack-stack/spack-stack-1.9.1/envs/gw-intel-2023.2.0/install/intel/2023.2.0/wgrib2-3.6.0-fjguk3g/bin/wgrib2") +prepend_path("PATH", "/autofs/ncrc-svm1_proj/epic/c6/spack-stack/spack-stack-1.9.1/envs/gw-intel-2023.2.0/install/intel/2023.2.0/wgrib2-3.6.0-fjguk3g/bin") + +unload("cray-libsci") + +whatis("Description: GFS run setup environment") diff --git a/modulefiles/gw_run.hera.lua b/modulefiles/gw_run.hera.lua new file mode 100644 index 00000000000..d88f3364879 --- /dev/null +++ b/modulefiles/gw_run.hera.lua @@ -0,0 +1,31 @@ +help([[ +Load environment to run GFS on Hera +]]) + +-- Test that HOMEgfs is set. +-- If not, load_gw_run_modules.sh was not sourced to load this module. +local homegfssdir=os.getenv("HOMEgfs") or "None" +if (homegfssdir == "None") then + LmodError("FATAL ERROR HOMEgfs variable is unset.\n" .. + "Please \"source ush/load_gw_run_modules.sh\" rather than loading this module directly.\n") +end + +load("gw_run.common") + +load(pathJoin("hpss", (os.getenv("hpss_ver") or "None"))) +load(pathJoin("gempak", (os.getenv("gempak_ver") or "None"))) +load(pathJoin("perl", (os.getenv("perl_ver") or "None"))) + +-- Point to the wgrib2 executable with ipolates +setenv("WGRIB2", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/intel/2021.5.0/wgrib2-2.0.8-nauzcdx/bin/wgrib2") +prepend_path("PATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/intel/2021.5.0/wgrib2-2.0.8-nauzcdx/bin/wgrib2") + +prepend_path("MODULEPATH", pathJoin("/scratch3/NCEPDEV/global/role.glopara/git_hera/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) + +prepend_path("MODULEPATH", pathJoin("/scratch3/NCEPDEV/global/role.glopara/git_hera/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) + +whatis("Description: GFS run environment") + +load(pathJoin("imagemagick", (os.getenv("imagemagick_ver") or "None"))) diff --git a/modulefiles/gw_run.hercules.lua b/modulefiles/gw_run.hercules.lua new file mode 100644 index 00000000000..8400b523344 --- /dev/null +++ b/modulefiles/gw_run.hercules.lua @@ -0,0 +1,28 @@ +help([[ +Load environment to run GFS on Hercules +]]) + +-- Test that HOMEgfs is set. +-- If not, load_gw_run_modules.sh was not sourced to load this module. +local homegfssdir=os.getenv("HOMEgfs") or "None" +if (homegfssdir == "None") then + LmodError("FATAL ERROR HOMEgfs variable is unset.\n" .. + "Please \"source ush/load_gw_run_modules.sh\" rather than loading this module directly.\n") +end + +load("gw_run.common") +load(pathJoin("intel-oneapi-mkl", (os.getenv("intel_mkl_ver") or "None"))) +load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) + +-- Set the path for the Sven executables +append_path("PATH", pathJoin((os.getenv("sven_root_path") or "None"), "bin")) +-- Load globus-cli for Globus-HPSS archiving +load("globus-cli") + +prepend_path("MODULEPATH", pathJoin("/work2/noaa/global/role-global/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) + +prepend_path("MODULEPATH", pathJoin("/work2/noaa/global/role-global/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) + +whatis("Description: GFS run environment") diff --git a/modulefiles/gw_run.noaacloud.lua b/modulefiles/gw_run.noaacloud.lua new file mode 100644 index 00000000000..4a8dd932128 --- /dev/null +++ b/modulefiles/gw_run.noaacloud.lua @@ -0,0 +1,26 @@ +help([[ +Load environment to run GFS on NOAA cloud +]]) + +-- Test that HOMEgfs is set. +-- If not, load_gw_run_modules.sh was not sourced to load this module. +local homegfssdir=os.getenv("HOMEgfs") or "None" +if (homegfssdir == "None") then + LmodError("FATAL ERROR HOMEgfs variable is unset.\n" .. + "Please \"source ush/load_gw_run_modules.sh\" rather than loading this module directly.\n") +end + +load(pathJoin("perl", (os.getenv("perl_ver") or "None"))) +load(pathJoin("mkl", (os.getenv("mkl_ver") or "None"))) + +load("gw_run.common") + +prepend_path("MODULEPATH", pathJoin("/contrib/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) + +prepend_path("MODULEPATH", pathJoin("/contrib/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) + +load(pathJoin("imagemagick", (os.getenv("imagemagick_ver") or "None"))) + +whatis("Description: GFS run environment") diff --git a/modulefiles/gw_run.orion.lua b/modulefiles/gw_run.orion.lua new file mode 100644 index 00000000000..6adbdbef66c --- /dev/null +++ b/modulefiles/gw_run.orion.lua @@ -0,0 +1,28 @@ +help([[ +Load environment to run GFS on Orion +]]) + +-- Test that HOMEgfs is set. +-- If not, load_gw_run_modules.sh was not sourced to load this module. +local homegfssdir=os.getenv("HOMEgfs") or "None" +if (homegfssdir == "None") then + LmodError("FATAL ERROR HOMEgfs variable is unset.\n" .. + "Please \"source ush/load_gw_run_modules.sh\" rather than loading this module directly.\n") +end + +load("gw_run.common") +load(pathJoin("intel-oneapi-mkl", (os.getenv("intel_mkl_ver") or "None"))) +load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) + +-- Set the path for the Sven executables +append_path("PATH", pathJoin((os.getenv("sven_root_path") or "None"), "bin")) +-- Load globus-cli for Globus-HPSS archiving +load("globus-cli") + +prepend_path("MODULEPATH", pathJoin("/work2/noaa/global/role-global/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) + +prepend_path("MODULEPATH", pathJoin("/work2/noaa/global/role-global/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) + +whatis("Description: GFS run environment") diff --git a/modulefiles/gw_run.ursa.lua b/modulefiles/gw_run.ursa.lua new file mode 100644 index 00000000000..1fcb7e639ec --- /dev/null +++ b/modulefiles/gw_run.ursa.lua @@ -0,0 +1,24 @@ +help([[ +Load environment to run GFS on Ursa +]]) + +-- Test that HOMEgfs is set. +-- If not, load_gw_run_modules.sh was not sourced to load this module. +local homegfssdir=os.getenv("HOMEgfs") or "None" +if (homegfssdir == "None") then + LmodError("FATAL ERROR HOMEgfs variable is unset.\n" .. + "Please \"source ush/load_gw_run_modules.sh\" rather than loading this module directly.\n") +end + +load("gw_run.common") +load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) + +prepend_path("MODULEPATH", pathJoin("/scratch3/NCEPDEV/global/role.glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) + +prepend_path("MODULEPATH", pathJoin("/scratch3/NCEPDEV/global/role.glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) + +whatis("Description: GFS run environment") + +load(pathJoin("imagemagick", (os.getenv("imagemagick_ver") or "None"))) diff --git a/modulefiles/module_base.wcoss2.lua b/modulefiles/gw_run.wcoss2.lua similarity index 81% rename from modulefiles/module_base.wcoss2.lua rename to modulefiles/gw_run.wcoss2.lua index 23e5d46661e..458fab5229d 100644 --- a/modulefiles/module_base.wcoss2.lua +++ b/modulefiles/gw_run.wcoss2.lua @@ -19,8 +19,14 @@ load(pathJoin("gsl", (os.getenv("gsl_ver") or "None"))) load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) load(pathJoin("imagemagick", (os.getenv("imagemagick_ver") or "None"))) -load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) -load(pathJoin("netcdf", (os.getenv("netcdf_ver") or "None"))) +load(pathJoin("hdf5-D", (os.getenv("hdf5_ver") or "None"))) +load(pathJoin("pnetcdf-D", (os.getenv("pnetcdf_ver") or "None"))) +load(pathJoin("netcdf-D", (os.getenv("netcdf_ver") or "None"))) +load(pathJoin("esmf-D", (os.getenv("esmf_ver") or "None"))) + +-- The cray library path for C MPI libraries (needed by C-only programs using netCDF) +local cray_lib_path=os.getenv("CRAY_LD_LIBRARY_PATH") or "" +prepend_path("LD_LIBRARY_PATH", cray_lib_path) load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) @@ -28,7 +34,7 @@ load(pathJoin("grib_util", (os.getenv("grib_util_ver") or "None"))) load(pathJoin("bufr_dump", (os.getenv("bufr_dump_ver") or "None"))) load(pathJoin("util_shared", (os.getenv("util_shared_ver") or "None"))) load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None"))) -load(pathJoin("ncdiag", (os.getenv("ncdiag_ver") or "None"))) +load(pathJoin("ncdiag-A", (os.getenv("ncdiag_ver") or "None"))) load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) diff --git a/modulefiles/module_gwsetup.gaeac6.lua b/modulefiles/gw_setup.gaeac6.lua similarity index 84% rename from modulefiles/module_gwsetup.gaeac6.lua rename to modulefiles/gw_setup.gaeac6.lua index e665d61b5ab..a781b105bb1 100644 --- a/modulefiles/module_gwsetup.gaeac6.lua +++ b/modulefiles/gw_setup.gaeac6.lua @@ -5,10 +5,10 @@ Load environment to run GFS workflow setup scripts on Gaea C6 prepend_path("MODULEPATH", "/autofs/ncrc-svm1_proj/hurr1/hafs/shared/modulefiles") load(pathJoin("rocoto", "1.3.7_fix")) -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.9.2/envs/ue-intel-2023.2.0/install/modulefiles/Core") local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" -local python_ver=os.getenv("python_ver") or "3.10.13" +local python_ver=os.getenv("python_ver") or "3.11.7" local cmake_ver=os.getenv("cmake_ver") or "3.27.9" load(pathJoin("cmake", cmake_ver)) diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/gw_setup.hera.lua similarity index 51% rename from modulefiles/module_gwsetup.hera.lua rename to modulefiles/gw_setup.hera.lua index c373e784ae1..56f8c50e665 100644 --- a/modulefiles/module_gwsetup.hera.lua +++ b/modulefiles/gw_setup.hera.lua @@ -4,14 +4,14 @@ Load environment to run GFS workflow setup scripts on Hera load(pathJoin("rocoto")) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.9.2/envs/ue-oneapi-2024.2.1/install/modulefiles/Core") -local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" -local python_ver=os.getenv("python_ver") or "3.11.6" +local stack_oneapi_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" +local python_ver=os.getenv("python_ver") or "3.11.7" local cmake_ver=os.getenv("cmake_ver") or "3.28.1" load(pathJoin("cmake", cmake_ver)) -load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-oneapi", stack_oneapi_ver)) load(pathJoin("python", python_ver)) load("py-jinja2") load("py-pyyaml") diff --git a/modulefiles/gw_setup.hercules.lua b/modulefiles/gw_setup.hercules.lua new file mode 100644 index 00000000000..fdc3a02c9d5 --- /dev/null +++ b/modulefiles/gw_setup.hercules.lua @@ -0,0 +1,21 @@ +help([[ +Load environment to run GFS workflow ci scripts on Hercules +]]) + +load(pathJoin("contrib","0.1")) +load(pathJoin("rocoto","1.3.7")) + +prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.2/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") + +local stack_oneapi_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" +local python_ver=os.getenv("python_ver") or "3.11.7" +local cmake_ver=os.getenv("cmake_ver") or "3.27.9" + +load(pathJoin("stack-oneapi", stack_oneapi_ver)) +load(pathJoin("stack-python", python_ver)) +load("py-jinja2") +load("py-pyyaml") +load("py-numpy") +try_load("globus-cli") + +whatis("Description: GFS run setup environment") diff --git a/modulefiles/gw_setup.noaacloud.lua b/modulefiles/gw_setup.noaacloud.lua new file mode 100644 index 00000000000..f57ac4f4866 --- /dev/null +++ b/modulefiles/gw_setup.noaacloud.lua @@ -0,0 +1,21 @@ +help([[ +Load environment to run GFS workflow setup scripts on noaacloud +]]) + +load("rocoto") + +prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.9.2/envs/ue-oneapi-2024.2.1/install/modulefiles/Core") + +local stack_oneapi_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" +local stack_mpi_ver=os.getenv("stack_mpi_ver") or "2021.13" +local cmake_ver=os.getenv("cmake_ver") or "3.27.9" + +load(pathJoin("stack-oneapi", stack_oneapi_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_mpi_ver)) +load(pathJoin("cmake", cmake_ver)) + +load("py-jinja2") +load("py-pyyaml") +load("py-numpy") + +whatis("Description: GFS run setup environment") diff --git a/modulefiles/module_gwsetup.orion.lua b/modulefiles/gw_setup.orion.lua similarity index 52% rename from modulefiles/module_gwsetup.orion.lua rename to modulefiles/gw_setup.orion.lua index a15ca221c6a..1580cc57369 100644 --- a/modulefiles/module_gwsetup.orion.lua +++ b/modulefiles/gw_setup.orion.lua @@ -6,13 +6,13 @@ prepend_path("MODULEPATH", "/apps/modulefiles/core") load(pathJoin("contrib","0.1")) load(pathJoin("rocoto","1.3.7")) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/gsi-addon-env-rocky9/install/modulefiles/Core") +prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.2/envs/ue-oneapi-2024.1.0/install/modulefiles/Core") -local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" -local python_ver=os.getenv("python_ver") or "3.11.6" -local cmake_ver=os.getenv("cmake_ver") or "3.23.1" +local stack_oneapi_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" +local python_ver=os.getenv("python_ver") or "3.11.7" +local cmake_ver=os.getenv("cmake_ver") or "3.27.9" -load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-oneapi", stack_oneapi_ver)) load(pathJoin("python", python_ver)) load("py-jinja2") load("py-pyyaml") diff --git a/modulefiles/gw_setup.ursa.lua b/modulefiles/gw_setup.ursa.lua new file mode 100644 index 00000000000..c7706764ff7 --- /dev/null +++ b/modulefiles/gw_setup.ursa.lua @@ -0,0 +1,20 @@ +help([[ +Load environment to run GFS workflow setup scripts on Ursa +]]) + +load(pathJoin("rocoto")) + +prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.9.2/envs/ue-oneapi-2024.2.1/install/modulefiles/Core") + +local stack_oneapi_ver=os.getenv("stack_oneapi_ver") or "2024.2.1" +local python_ver=os.getenv("python_ver") or "3.11.7" +local cmake_ver=os.getenv("cmake_ver") or "3.27.9" + +load(pathJoin("stack-oneapi", stack_oneapi_ver)) +load(pathJoin("stack-python", python_ver)) +load(pathJoin("cmake", cmake_ver)) +load("py-jinja2") +load("py-pyyaml") +load("py-numpy") + +whatis("Description: GFS run setup environment") diff --git a/modulefiles/module_gwsetup.wcoss2.lua b/modulefiles/gw_setup.wcoss2.lua similarity index 100% rename from modulefiles/module_gwsetup.wcoss2.lua rename to modulefiles/gw_setup.wcoss2.lua diff --git a/modulefiles/module_base.gaeac5.lua b/modulefiles/module_base.gaeac5.lua deleted file mode 100644 index 9208c8c4d89..00000000000 --- a/modulefiles/module_base.gaeac5.lua +++ /dev/null @@ -1,52 +0,0 @@ -help([[ -Load environment to run GFS on Gaea C5 -]]) - -local spack_mod_path=(os.getenv("spack_mod_path") or "None") -prepend_path("MODULEPATH", spack_mod_path) - -load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) -load(pathJoin("stack-cray-mpich", (os.getenv("stack_cray_mpich_ver") or "None"))) -load(pathJoin("python", (os.getenv("python_ver") or "None"))) - -load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) -load(pathJoin("libpng", (os.getenv("libpng_ver") or "None"))) -load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) -load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) -load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) -load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) -load(pathJoin("esmf", (os.getenv("esmf_ver") or "None"))) -load(pathJoin("perlbrew", (os.getenv("perl_ver") or "None"))) - -load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) -load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) -load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None"))) -load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None"))) -load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None"))) -load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) -load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) -load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) -load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) -load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) -load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) -load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) -load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) -load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None"))) -load(pathJoin("met", (os.getenv("met_ver") or "None"))) -load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) -load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) - -setenv("WGRIB2","wgrib2") -setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) - -prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) -load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) - -prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) -load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) - -local hsi_mod_path=(os.getenv("hsi_mod_path") or "None") -append_path("MODULEPATH", hsi_mod_path) -load(pathJoin("hsi", (os.getenv("hsi_ver") or "None"))) - -whatis("Description: GFS run setup environment") diff --git a/modulefiles/module_base.gaeac6.lua b/modulefiles/module_base.gaeac6.lua deleted file mode 100644 index 9da9be98922..00000000000 --- a/modulefiles/module_base.gaeac6.lua +++ /dev/null @@ -1,58 +0,0 @@ -help([[ -Load environment to run GFS on Gaea C6 -]]) - -local spack_mod_path=(os.getenv("spack_mod_path") or "None") -prepend_path("MODULEPATH", spack_mod_path) - -load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) -load(pathJoin("stack-cray-mpich", (os.getenv("stack_cray_mpich_ver") or "None"))) -load(pathJoin("python", (os.getenv("python_ver") or "None"))) - -load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) -load(pathJoin("libpng", (os.getenv("libpng_ver") or "None"))) -load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) -load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) -load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) -load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) -load(pathJoin("esmf", (os.getenv("esmf_ver") or "None"))) -load(pathJoin("perlbrew", (os.getenv("perl_ver") or "None"))) - -load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) -load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) -load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None"))) -load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None"))) -load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None"))) -load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) -load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) -load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) -load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) -load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) -load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) -load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) -load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) -load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None"))) -load(pathJoin("met", (os.getenv("met_ver") or "None"))) -load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) -load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) - -setenv("WGRIB2","wgrib2") - --- Stop gap fix for wgrib with spack-stack 1.6.0 --- TODO Remove this when spack-stack issue #1097 is resolved -setenv("WGRIB","wgrib") -setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) - -prepend_path("MODULEPATH", pathJoin("/gpfs/f6/drsa-precip3/world-shared/role.glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) -load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) - -prepend_path("MODULEPATH", pathJoin("/gpfs/f6/drsa-precip3/world-shared/role.glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) -load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) - -local hsi_mod_path=(os.getenv("hsi_mod_path") or "None") -append_path("MODULEPATH", hsi_mod_path) -load(pathJoin("hsi", (os.getenv("hsi_ver") or "None"))) - -unload("cray-libsci") - -whatis("Description: GFS run setup environment") diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua deleted file mode 100644 index 8b4eec2aef4..00000000000 --- a/modulefiles/module_base.hera.lua +++ /dev/null @@ -1,59 +0,0 @@ -help([[ -Load environment to run GFS on Hera -]]) - -local spack_mod_path=(os.getenv("spack_mod_path") or "None") -prepend_path("MODULEPATH", spack_mod_path) - -load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) -load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None"))) -load(pathJoin("python", (os.getenv("python_ver") or "None"))) - -load(pathJoin("hpss", (os.getenv("hpss_ver") or "None"))) -load(pathJoin("gempak", (os.getenv("gempak_ver") or "None"))) -load(pathJoin("ncl", (os.getenv("ncl_ver") or "None"))) -load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) -load(pathJoin("libpng", (os.getenv("libpng_ver") or "None"))) -load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) -load(pathJoin("R", (os.getenv("R_ver") or "None"))) -load(pathJoin("perl", (os.getenv("perl_ver") or "None"))) - -load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) -load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) -load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) -load(pathJoin("esmf", (os.getenv("esmf_ver") or "None"))) - -load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) -load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) -load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None"))) -load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None"))) -load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None"))) -load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) -load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) -load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) -load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) -load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) -load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) -load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) -load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) -load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None"))) -load(pathJoin("met", (os.getenv("met_ver") or "None"))) -load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) -load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) - -setenv("WGRIB2","wgrib2") - --- Stop gap fix for wgrib with spack-stack 1.6.0 --- TODO Remove this when spack-stack issue #1097 is resolved -setenv("WGRIB","wgrib") -setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) - -prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) -load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) - -prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) -load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) - -whatis("Description: GFS run environment") - -load(pathJoin("imagemagick", (os.getenv("imagemagick_ver") or "None"))) diff --git a/modulefiles/module_base.hercules.lua b/modulefiles/module_base.hercules.lua deleted file mode 100644 index 805e7edb73c..00000000000 --- a/modulefiles/module_base.hercules.lua +++ /dev/null @@ -1,56 +0,0 @@ -help([[ -Load environment to run GFS on Hercules -]]) - -local spack_mod_path=(os.getenv("spack_mod_path") or "None") -prepend_path("MODULEPATH", spack_mod_path) - -load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) -load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None"))) -load(pathJoin("intel-oneapi-mkl", (os.getenv("intel_mkl_ver") or "None"))) -load(pathJoin("python", (os.getenv("python_ver") or "None"))) - -load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) -load(pathJoin("libpng", (os.getenv("libpng_ver") or "None"))) -load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) - -load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) -load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) -load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) -load(pathJoin("esmf", (os.getenv("esmf_ver") or "None"))) - -load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) -load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) -load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None"))) -load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None"))) -load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None"))) -load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) -load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) -load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) -load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) -load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) -load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) -load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) -load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) -load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None"))) -load(pathJoin("met", (os.getenv("met_ver") or "None"))) -load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) -load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) - --- TODO remove this once we move to spack-stack 1.8.0+; as WGRIB2 will be assigned by the wgrib2 module -setenv("WGRIB2","wgrib2") - -setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) - --- Set the path for the Sven executables -append_path("PATH", pathJoin((os.getenv("sven_root_path") or "None"), "bin")) - -prepend_path("MODULEPATH", pathJoin("/work2/noaa/global/role-global/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) -load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) - -prepend_path("MODULEPATH", pathJoin("/work2/noaa/global/role-global/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) -load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) - -load("globus-cli") - -whatis("Description: GFS run environment") diff --git a/modulefiles/module_base.noaacloud.lua b/modulefiles/module_base.noaacloud.lua deleted file mode 100644 index bd10d0eabf1..00000000000 --- a/modulefiles/module_base.noaacloud.lua +++ /dev/null @@ -1,52 +0,0 @@ -help([[ -Load environment to run GFS on noaacloud -]]) - -local spack_mod_path=(os.getenv("spack_mod_path") or "None") -prepend_path("MODULEPATH", spack_mod_path) - -load("gnu") -load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) -load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None"))) -unload("gnu") - -load(pathJoin("python", (os.getenv("python_ver") or "None"))) - -load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) -load(pathJoin("libpng", (os.getenv("libpng_ver") or "None"))) -load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) ---load(pathJoin("R", (os.getenv("R_ver") or "None"))) -load(pathJoin("perl", (os.getenv("perl_ver") or "None"))) - -load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) -load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) -load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) - -load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) -load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) -load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None"))) -load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None"))) -load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None"))) -load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) -load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) -load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) -load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) -load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) -load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) -load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) -load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) -load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None"))) ---load(pathJoin("met", (os.getenv("met_ver") or "None"))) ---load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) -load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) - -setenv("WGRIB2","wgrib2") -setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) - -prepend_path("MODULEPATH", "/contrib/git/prepobs/modulefiles") -load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) - -prepend_path("MODULEPATH", "/contrib/git/Fit2Obs/modulefiles") -load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) - -whatis("Description: GFS run environment") diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua deleted file mode 100644 index 4f9d0de1069..00000000000 --- a/modulefiles/module_base.orion.lua +++ /dev/null @@ -1,55 +0,0 @@ -help([[ -Load environment to run GFS on Orion -]]) - -local spack_mod_path=(os.getenv("spack_mod_path") or "None") -prepend_path("MODULEPATH", spack_mod_path) - -load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) -load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None"))) -load(pathJoin("python", (os.getenv("python_ver") or "None"))) - -load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) -load(pathJoin("libpng", (os.getenv("libpng_ver") or "None"))) -load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) - -load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) -load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) -load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) -load(pathJoin("esmf", (os.getenv("esmf_ver") or "None"))) - -load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) -load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) -load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None"))) -load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None"))) -load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None"))) -load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) -load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) -load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) -load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) -load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) -load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) -load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) -load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) -load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None"))) -load(pathJoin("met", (os.getenv("met_ver") or "None"))) -load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) -load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) - --- TODO remove this once we move to spack-stack 1.8.0+; as WGRIB2 will be assigned by the wgrib2 module -setenv("WGRIB2","wgrib2") - -setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) - --- Set the path for the Sven executables -append_path("PATH", pathJoin((os.getenv("sven_root_path") or "None"), "bin")) - -prepend_path("MODULEPATH", pathJoin("/work2/noaa/global/role-global/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) -load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) - -prepend_path("MODULEPATH", pathJoin("/work2/noaa/global/role-global/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) -load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) - -load("globus-cli") - -whatis("Description: GFS run environment") diff --git a/modulefiles/module_gwci.hera.lua b/modulefiles/module_gwci.hera.lua deleted file mode 100644 index 75b972b69ba..00000000000 --- a/modulefiles/module_gwci.hera.lua +++ /dev/null @@ -1,15 +0,0 @@ -help([[ -Load environment to run GFS workflow setup scripts on Hera -]]) - -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core") - -load(pathJoin("stack-intel", os.getenv("2021.5.0"))) -load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.5.1"))) - -load(pathJoin("netcdf-c", os.getenv("4.9.2"))) -load(pathJoin("netcdf-fortran", os.getenv("4.6.1"))) -load(pathJoin("nccmp","1.9.0.1")) -load(pathJoin("wgrib2", "2.0.8")) - -whatis("Description: GFS run setup CI environment") diff --git a/modulefiles/module_gwci.hercules.lua b/modulefiles/module_gwci.hercules.lua deleted file mode 100644 index 179bbef1149..00000000000 --- a/modulefiles/module_gwci.hercules.lua +++ /dev/null @@ -1,15 +0,0 @@ -help([[ -Load environment to run GFS workflow ci scripts on Hercules -]]) - -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") - -load(pathJoin("stack-intel", os.getenv("2021.9.0"))) -load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.9.0"))) - -load(pathJoin("netcdf-c", os.getenv("4.9.2"))) -load(pathJoin("netcdf-fortran", os.getenv("4.6.0"))) -load(pathJoin("nccmp","1.9.0.1")) -load(pathJoin("wgrib2", "3.1.1")) - -whatis("Description: GFS run ci top-level sripts environment") diff --git a/modulefiles/module_gwci.noaacloud.lua b/modulefiles/module_gwci.noaacloud.lua deleted file mode 100644 index 2ac284ef85b..00000000000 --- a/modulefiles/module_gwci.noaacloud.lua +++ /dev/null @@ -1,15 +0,0 @@ -help([[ -Load environment to run GFS workflow setup scripts on noaacloud -]]) - -prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/ue-env/install/modulefiles/Core") - -load(pathJoin("stack-intel", os.getenv("2021.10.0"))) -load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.10.0"))) - -load(pathJoin("netcdf-c", os.getenv("4.9.2"))) -load(pathJoin("netcdf-fortran", os.getenv("4.6.1"))) -load(pathJoin("nccmp","1.9.0.1")) -load(pathJoin("wgrib2", "2.0.8")) - -whatis("Description: GFS run setup CI environment") diff --git a/modulefiles/module_gwci.orion.lua b/modulefiles/module_gwci.orion.lua deleted file mode 100644 index e2213932d75..00000000000 --- a/modulefiles/module_gwci.orion.lua +++ /dev/null @@ -1,15 +0,0 @@ -help([[ -Load environment to run GFS workflow ci scripts on Orion -]]) - -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/gsi-addon-env-rocky9/install/modulefiles/Core") - -load(pathJoin("stack-intel", os.getenv("2021.9.0"))) -load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.9.0"))) - -load(pathJoin("netcdf-c", os.getenv("4.9.2"))) -load(pathJoin("netcdf-fortran", os.getenv("4.6.1"))) -load(pathJoin("nccmp","1.9.0.1")) -load(pathJoin("wgrib2", "2.0.8")) - -whatis("Description: GFS run ci top-level sripts environment") diff --git a/modulefiles/module_gwci.wcoss2.lua b/modulefiles/module_gwci.wcoss2.lua deleted file mode 100644 index 50eaeefee00..00000000000 --- a/modulefiles/module_gwci.wcoss2.lua +++ /dev/null @@ -1,8 +0,0 @@ -help([[ -Load environment to run GFS workflow ci scripts on WCOSS2 -]]) - -prepend_path("MODULEPATH", "/apps/ops/para/nco/modulefiles/core") -load(pathJoin("gh","2.28.0")) - -whatis("Description: GFS run setup ci environment") diff --git a/modulefiles/module_gwsetup.gaeac5.lua b/modulefiles/module_gwsetup.gaeac5.lua deleted file mode 100644 index 0c1691d30b2..00000000000 --- a/modulefiles/module_gwsetup.gaeac5.lua +++ /dev/null @@ -1,20 +0,0 @@ -help([[ -Load environment to run GFS workflow setup scripts on Gaea C5 -]]) - -prepend_path("MODULEPATH", "/ncrc/proj/epic/rocoto/modulefiles") -load(pathJoin("rocoto")) - -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") - -local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" -local python_ver=os.getenv("python_ver") or "3.10.13" - -load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("python", python_ver)) -load("py-jinja2") -load("py-pyyaml") -load("py-numpy") -load("git-lfs") - -whatis("Description: GFS run setup environment") diff --git a/modulefiles/module_gwsetup.hercules.lua b/modulefiles/module_gwsetup.hercules.lua deleted file mode 100644 index c0bc7760a35..00000000000 --- a/modulefiles/module_gwsetup.hercules.lua +++ /dev/null @@ -1,21 +0,0 @@ -help([[ -Load environment to run GFS workflow ci scripts on Hercules -]]) - -load(pathJoin("contrib","0.1")) -load(pathJoin("rocoto","1.3.7")) - -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") - -local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" -local python_ver=os.getenv("python_ver") or "3.11.6" -local cmake_ver=os.getenv("cmake_ver") or "3.23.1" - -load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("python", python_ver)) -load("py-jinja2") -load("py-pyyaml") -load("py-numpy") -try_load("globus-cli") - -whatis("Description: GFS run setup environment") diff --git a/modulefiles/module_gwsetup.noaacloud.lua b/modulefiles/module_gwsetup.noaacloud.lua deleted file mode 100644 index e2aa4050a34..00000000000 --- a/modulefiles/module_gwsetup.noaacloud.lua +++ /dev/null @@ -1,21 +0,0 @@ -help([[ -Load environment to run GFS workflow setup scripts on noaacloud -]]) - -load(pathJoin("rocoto")) - -prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/ue-intel/install/modulefiles/Core") - -load("gnu") -local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.10.0" -local stack_mpi_ver=os.getenv("stack_mpi_ver") or "2021.10.0" - -load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("stack-intel-oneapi-mpi", stack_mpi_ver)) -unload("gnu") - -load("py-jinja2") -load("py-pyyaml") -load("py-numpy") - -whatis("Description: GFS run setup environment") diff --git a/parm/ufs/gocart/CA2G_instance_CA.bc.rc b/parm/ufs/gocart/CA2G_instance_CA.bc.rc index f6411e2c9d7..9ac462fa3e6 100644 --- a/parm/ufs/gocart/CA2G_instance_CA.bc.rc +++ b/parm/ufs/gocart/CA2G_instance_CA.bc.rc @@ -1,5 +1,5 @@ # -# Resource file for Black Carbon parameters. +# Resource file for Black Carbon parameters. # nbins: 2 @@ -16,8 +16,22 @@ aviation_vertical_layers: 0.0 100.0 9.0e3 10.0e3 # Initially hydrophobic portion hydrophobic_fraction: 0.8 +# Rate of conversion of hydrophobic to hydrophilic [days] +time_days_hydrophobic_to_hydrophilic: 2.5 + +# Rate of chemical destruction of carbon species [days] +time_days_chemical_destruction: -1. -1. + +# Wet Removal Scheme Option | gocart, ufs +wet_removal_scheme: ufs # default value + +# Rainout efficiency per bin for wet_removal_scheme == ufs +fwet_ice: 0.0 1.0 +fwet_snow: 0.0 1.0 +fwet_rain: 0.0 1.0 + # Scavenging efficiency per bin [km-1] (NOT USED UNLESS RAS IS CALLED) -fscav: 0.0 0.4 +fscav: 0.0 0.4 # Dry particle density [kg m-3] particle_density: 1800 1800 diff --git a/parm/ufs/gocart/CA2G_instance_CA.br.rc b/parm/ufs/gocart/CA2G_instance_CA.br.rc index 41360831f33..e983240212b 100644 --- a/parm/ufs/gocart/CA2G_instance_CA.br.rc +++ b/parm/ufs/gocart/CA2G_instance_CA.br.rc @@ -21,6 +21,18 @@ pom_ca_ratio: 1.8 # Initially hydrophobic portion hydrophobic_fraction: 0.5 +# Rate of conversion of hydrophobic to hydrophilic [days] +time_days_hydrophobic_to_hydrophilic: 2.5 + +# Rate of chemical destruction of carbon species [days] +time_days_chemical_destruction: -1. -1. + +# Wet Removal Scheme Option | gocart, ufs +wet_removal_scheme: ufs +fwet_ice: 0.0 0.4 +fwet_snow: 0.0 0.4 +fwet_rain: 0.0 0.4 + # Scavenging efficiency per bin [km-1] (NOT USED UNLESS RAS IS CALLED) fscav: 0.0 0.4 diff --git a/parm/ufs/gocart/CA2G_instance_CA.oc.rc b/parm/ufs/gocart/CA2G_instance_CA.oc.rc index 6e3f5ef978a..f24d65e4c47 100644 --- a/parm/ufs/gocart/CA2G_instance_CA.oc.rc +++ b/parm/ufs/gocart/CA2G_instance_CA.oc.rc @@ -1,5 +1,5 @@ # -# Resource file for Organic Carbon parameters. +# Resource file for Organic Carbon parameters. # aerosol_radBands_optics_file: ExtData/optics/opticsBands_OC.v1_3.RRTMG.nc @@ -26,8 +26,22 @@ rhFlag: 0 # Initially hydrophobic portion hydrophobic_fraction: 0.5 +# Rate of conversion of hydrophobic to hydrophilic [days] +time_days_hydrophobic_to_hydrophilic: 2.5 + +# Rate of chemical destruction of carbon species [days] +time_days_chemical_destruction: -1. -1. + +# Wet Removal Scheme Option | gocart, ufs +wet_removal_scheme: ufs + +# Rainout efficiency per bin for wet_removal_scheme == ufs +fwet_ice: 0.0 1.0 +fwet_snow: 0.0 1.0 +fwet_rain: 0.0 1.0 + # Scavenging efficiency per bin [km-1] (NOT USED UNLESS RAS IS CALLED) -fscav: 0.0 0.4 +fscav: 0.0 0.4 # Dry particle density [kg m-3] particle_density: 1800 1800 @@ -43,6 +57,6 @@ sigma: 2.20 2.20 pressure_lid_in_hPa: 0.01 -nbins: 2 +nbins: 2 point_emissions_srcfilen: /dev/null diff --git a/parm/ufs/gocart/DU2G_instance_DU.rc b/parm/ufs/gocart/DU2G_instance_DU.rc index 6c30cdf06b8..8001798189d 100644 --- a/parm/ufs/gocart/DU2G_instance_DU.rc +++ b/parm/ufs/gocart/DU2G_instance_DU.rc @@ -1,5 +1,5 @@ # -# Resource file Dust parameters. +# Resource file Dust parameters. # aerosol_radBands_optics_file: ExtData/optics/opticsBands_DU.v15_3.RRTMG.nc @@ -22,6 +22,17 @@ Ch_DU: 0.2 0.2 0.07 0.07 0.07 0.056 # Scavenging efficiency per bin [km-1] fscav: 0.2 0.2 0.2 0.2 0.2 # +# clay fraction scaling factor +soil_clay_factor: 1.0 + +# Wet Removal Scheme Option | gocart, ufs +wet_removal_scheme: ufs + +# Rainout efficiency per bin for wet_removal_scheme == ufs +fwet_ice: 0.8 0.8 0.8 1.0 1.0 +fwet_snow: 0.8 0.8 0.8 1.0 1.0 +fwet_rain: 0.8 0.8 0.8 1.0 1.0 + # Molecular weight of species [kg mole-1] molecular_weight: 0.1 0.1 0.1 0.1 0.1 @@ -31,7 +42,7 @@ fnum: 2.45e14 3.28e13 6.52e12 9.89e11 1.76e11 rhFlag: 0 # Maring settling velocity correction -maringFlag: .true. +maringFlag: .true. nbins: 5 @@ -41,8 +52,9 @@ pressure_lid_in_hPa: 0.01 emission_scheme: fengsha # choose among: fengsha, ginoux, k14 # FENGSHA settings -alpha: 0.1 +alpha: 0.16 gamma: 1.0 soil_moisture_factor: 1 soil_drylimit_factor: 1 vertical_to_horizontal_flux_ratio_limit: 2.e-04 +drag_partition_option: 2 \ No newline at end of file diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index 5d2ddc51021..1c138b4b312 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -7,13 +7,15 @@ TROPP 'Pa' Y N - 0.0 1.0 TROPP /dev/null:10000. #====== Dust Imports ================================================= -# FENGSHA input files. Note: regridding should be N or E - Use files with _FillValue != NaN +# FENGSHA input files. Note: regridding should be N or E - Use files with _FillValue != NaN DU_CLAY '1' Y E - none none clayfrac ExtData/nexus/FENGSHA/FENGSHA_2022_NESDIS_inputs_10km_v3.2.nc DU_SAND '1' Y E - none none sandfrac ExtData/nexus/FENGSHA/FENGSHA_2022_NESDIS_inputs_10km_v3.2.nc DU_SILT '1' Y E - none none siltfrac /dev/null -DU_SSM '1' Y E - none none sep ExtData/nexus/FENGSHA/FENGSHA_2022_NESDIS_inputs_10km_v3.2.nc -DU_RDRAG '1' Y E %y4-%m2-%d2t12:00:00 none none albedo_drag ExtData/nexus/FENGSHA/FENGSHA_2022_NESDIS_inputs_10km_v3.2.nc +DU_SSM '1' Y E - none none sep /dev/null:1.0 DU_UTHRES '1' Y E - none none uthres ExtData/nexus/FENGSHA/FENGSHA_2022_NESDIS_inputs_10km_v3.2.nc +DU_RDRAG '1' Y E %y4-%m2-%d2t12:00:00 none none PC ExtData/nexus/FENGSHA/FENGSHA_New_Method_NESDISv1.1_9km.nc +DU_GVF '1' Y E %y4-%m2-%d2T12:00:00 none none GVF ExtData/nexus/FENGSHA/FENGSHA_GVF_LAI2.nc +DU_LAI '1' Y E %y4-%m2-%d2T12:00:00 none none LAI ExtData/nexus/FENGSHA/FENGSHA_GVF_LAI2.nc #====== Sulfate Sources ================================================= # Anthropogenic (BF & FF) emissions -- allowed to input as two layers diff --git a/parm/ufs/gocart/SS2G_instance_SS.rc b/parm/ufs/gocart/SS2G_instance_SS.rc index d8faa3efa8a..5616616ea6c 100644 --- a/parm/ufs/gocart/SS2G_instance_SS.rc +++ b/parm/ufs/gocart/SS2G_instance_SS.rc @@ -14,7 +14,7 @@ radius_upper: 0.1 0.5 1.5 5.0 10.0 particle_density: 2200. 2200. 2200. 2200. 2200. # Scavenging efficiency per bin [km-1] -fscav: 0.4 0.4 0.4 0.4 0.4 +fscav: 0.4 0.4 0.4 0.4 0.4 # Emissions methods and scaling emission_scheme: 3 # 1 for Gong 2003, 2 for ... @@ -40,4 +40,12 @@ nbins: 5 pressure_lid_in_hPa: 0.01 +# Wet Removal Scheme Option | gocart, ufs +wet_removal_scheme: ufs + +# Rainout efficiency per bin for wet_removal_scheme == ufs +fwet_ice: 1.0 1.0 1.0 1.0 1.0 +fwet_snow: 1.0 1.0 1.0 1.0 1.0 +fwet_rain: 1.0 1.0 1.0 1.0 1.0 + diff --git a/parm/ufs/gocart/SU2G_instance_SU.rc b/parm/ufs/gocart/SU2G_instance_SU.rc index 79484b30683..2efa5ec49f8 100644 --- a/parm/ufs/gocart/SU2G_instance_SU.rc +++ b/parm/ufs/gocart/SU2G_instance_SU.rc @@ -8,7 +8,8 @@ aerosol_monochromatic_optics_file: ExtData/monochromatic/optics_SU.v1_3.nc nbins: 4 # Volcanic pointwise sources | degassing only | replace with path to historical volcanic emissions for scientific runs -volcano_srcfilen: ExtData/nexus/VOLCANO/v2021-09/so2_volcanic_emissions_CARN_v202005.degassing_only.rc +volcano_srcfilen_explosive: /dev/null +volcano_srcfilen_degassing: ExtData/nexus/VOLCANO/v2021-09/so2_volcanic_emissions_CARN_v202005.degassing_only.rc # Heights [m] of LTO, CDS and CRS aviation emissions layers aviation_vertical_layers: 0.0 100.0 9.0e3 10.0e3 diff --git a/sorc/build_compute.sh b/sorc/build_compute.sh index 50bc4252f94..edc5a50f87d 100755 --- a/sorc/build_compute.sh +++ b/sorc/build_compute.sh @@ -65,12 +65,17 @@ if [[ "${verbose}" == "YES" ]]; then set -x fi -# shellcheck disable=SC2155,SC2312 -HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/.." && pwd -P) +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" +HOMEgfs=$(cd "${script_dir}" && git rev-parse --show-toplevel) +# Needs to be exported for gw_setup.sh +export HOMEgfs echo "Sourcing global-workflow modules ..." source "${HOMEgfs}/dev/ush/gw_setup.sh" +# Un-export after gw_setup.sh +export -n HOMEgfs + cd "${HOMEgfs}/sorc" || exit 1 mkdir -p "${HOMEgfs}/sorc/logs" || exit 1 diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 41a5edc4a13..392cf872fb4 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 41a5edc4a135f5b25e469e0650a4e857fa00b219 +Subproject commit 392cf872fb47106571eacda241d832519ad4afef diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 7361f31a69e..4f2806c15dc 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 7361f31a69e9eaa599d708c592e3ac9df3ea1a9e +Subproject commit 4f2806c15dc49359e0eb02b497edb00cb2e0c2f4 diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 964bcc37597..16c19a2bca9 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 964bcc37597310d99d22627d769d5e5a82d4dd5a +Subproject commit 16c19a2bca9fe32cf64c3972f8f4fea229c31420 diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd index 08b7aedc178..d5380b7a81a 160000 --- a/sorc/gsi_monitor.fd +++ b/sorc/gsi_monitor.fd @@ -1 +1 @@ -Subproject commit 08b7aedc178e23290dda56fdc523d0f448739543 +Subproject commit d5380b7a81a51d60afae88e0ea4fb2e074d95d74 diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index 2e3b856a3e2..c5d3a513cfb 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit 2e3b856a3e2d47c92d114c67f9231331d1a20e07 +Subproject commit c5d3a513cfb3cf0c9878a83c2b4af017cceac5cf diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index bc69ddabeb3..f9d23c7a012 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -71,7 +71,7 @@ ${LINK_OR_COPY} "${HOMEgfs}/versions/run.${machine}.ver" "${HOMEgfs}/versions/ru #------------------------------ case "${machine}" in "wcoss2") FIX_DIR="/lfs/h2/emc/global/noscrub/emc.global/FIX/fix" ;; -"hera") FIX_DIR="/scratch3/NCEPDEV/global/role.glopara/fix" ;; +"hera" | "ursa") FIX_DIR="/scratch3/NCEPDEV/global/role.glopara/fix" ;; "orion") FIX_DIR="/work2/noaa/global/role-global/fix" ;; "hercules") FIX_DIR="/work2/noaa/global/role-global/fix" ;; "gaeac5") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; @@ -196,6 +196,11 @@ if [[ -s "atparse.bash" ]]; then fi ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/atparse.bash" . +# Temporarilly link load_gw_run_modules.sh to load_fv3gfs_modules.sh +# TODO: Use load_gw_run_modules.sh everywhere +cd "${HOMEgfs}/dev/ush" || exit 1 +${LINK_OR_COPY} "${HOMEgfs}/dev/ush/load_gw_run_modules.sh" load_fv3gfs_modules.sh + # add ufs_utils parm dir if [[ -d "${HOMEgfs}/sorc/ufs_utils.fd" ]]; then cd "${HOMEgfs}/parm" || exit 1 diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 1f0ab4322b0..d820b67f674 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 1f0ab4322b0c81e48a0db5a202a427d374e29be0 +Subproject commit d820b67f674e1636a9b55dd1ef780b22693fc873 diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 2dcbd1f8f71..4a982afe767 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 2dcbd1f8f719853650eea00198aa13b648cbe2b1 +Subproject commit 4a982afe767116271117c443e2aff215259e1623 diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 24fb2813e0f..8a3ad5c08c8 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -33,6 +33,10 @@ case $(hostname -f) in hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12 hecflow01) MACHINE_ID=hera ;; ### heraecflow01 + ufe0[1-9]) MACHINE_ID=ursa ;; ### ursa01-09 + ufe1[0-6]) MACHINE_ID=ursa ;; ### ursa10-16 + uecflow01) MACHINE_ID=ursa ;; ### ursaecflow01 + s4-submit.ssec.wisc.edu) MACHINE_ID=s4 ;; ### s4 fe[1-8]) MACHINE_ID=jet ;; ### jet01-8 @@ -64,7 +68,6 @@ MACHINE_ID=${MACHINE:-${MACHINE_ID}} if [[ "${MACHINE_ID}" != "UNKNOWN" ]]; then return fi - # Try searching based on paths since hostname may not match on compute nodes if [[ -d /lfs/h3 ]]; then # We are on NOAA Cactus or Dogwood @@ -76,12 +79,18 @@ elif [[ -d /mnt/lfs5 ]]; then # We are on NOAA Jet MACHINE_ID=jet elif [[ -d /scratch3 ]]; then - # We are on NOAA Hera - MACHINE_ID=hera + # We are on NOAA Hera or Ursa + mount=$(findmnt -n -o SOURCE /home) || true # /home doesn't exist on the GitHub runners + # TODO: When Hera is no longer supported, assume `/scratch3` means we're on Ursa + if [[ ${mount} =~ "ursa" ]]; then + MACHINE_ID=ursa + else + MACHINE_ID=hera + fi elif [[ -d /work ]]; then # We are on MSU Orion or Hercules mount=$(findmnt -n -o SOURCE /home) - if [[ ${mount} =~ "hercules" ]]; then + if [[ -n "${mount+0}" && ${mount} =~ "hercules" ]]; then MACHINE_ID=hercules else MACHINE_ID=orion diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 61948fa3995..4065c67e4cd 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -13,6 +13,16 @@ if [[ ${MACHINE_ID} = hera* ]] ; then module reset set -u +elif [[ ${MACHINE_ID} = ursa* ]] ; then + # We are on NOAA Ursa + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /apps/lmod/lmod/init/bash + fi + export LMOD_SYSTEM_DEFAULT_MODULES=lmod + set +u + module reset + set -u + elif [[ ${MACHINE_ID} = hercules* ]] ; then # We are on Hercules if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/ush/parsing_namelists_FV3.sh b/ush/parsing_namelists_FV3.sh index f1ba0a14477..b8d33f2b401 100755 --- a/ush/parsing_namelists_FV3.sh +++ b/ush/parsing_namelists_FV3.sh @@ -174,6 +174,8 @@ local IMP_PHYSICS=${imp_physics:-"99"} local default_dt_inner=$(( DELTIM/2 )) local IOVR=${iovr:-"3"} local LTAEROSOL=${ltaerosol:-".false."} +local MRAEROSOL=.false. +local LTHAILAWARE=.false. local LRADAR=${lradar:-".true."} local TTENDLIM=${ttendlim:-"-999"} local DT_INNER=${dt_inner:-"${default_dt_inner}"} diff --git a/versions/build.gaeac5.ver b/versions/build.gaeac5.ver deleted file mode 100644 index 681607855d4..00000000000 --- a/versions/build.gaeac5.ver +++ /dev/null @@ -1,5 +0,0 @@ -export stack_intel_ver=2023.2.0 -export stack_cray_mpich_ver=8.1.28 -export spack_env=gsi-addon-dev-fms-2024.01 -source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/build.gaeac6.ver b/versions/build.gaeac6.ver index ae3ef5e407b..d8c859809bd 100644 --- a/versions/build.gaeac6.ver +++ b/versions/build.gaeac6.ver @@ -1,5 +1,10 @@ -export stack_intel_ver=2023.2.0 -export stack_cray_mpich_ver=8.1.29 -export spack_env=gsi-addon-dev-fms-2024.01 +#!/usr/bin/env bash +export stack_compiler=intel +export stack_compiler_ver=2023.2.0 + +export stack_mpi=cray-mpich +export stack_mpi_ver=8.1.30 + +spack_stack_root="/ncrc/proj/epic/spack-stack" + source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/ncrc/proj/epic/spack-stack/c6/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/build.hera.ver b/versions/build.hera.ver index da7be231d09..b35f8ee5d9a 100644 --- a/versions/build.hera.ver +++ b/versions/build.hera.ver @@ -1,5 +1,10 @@ -export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-dev-fms-2024.01 +#!/usr/bin/env bash +export stack_compiler=oneapi +export stack_compiler_ver=2024.2.1 + +export stack_mpi=intel-oneapi-mpi +export stack_mpi_ver=2021.13 + +spack_stack_root="/contrib/spack-stack" + source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/build.hercules.ver b/versions/build.hercules.ver index 17dd60abb10..0ca97fa8cfc 100644 --- a/versions/build.hercules.ver +++ b/versions/build.hercules.ver @@ -1,6 +1,13 @@ -export stack_intel_ver=2021.9.0 -export stack_impi_ver=2021.9.0 -export intel_mkl_ver=2023.1.0 -export spack_env=gsi-addon-dev-fms-2024.01 +#!/usr/bin/env bash +export stack_compiler=oneapi +export stack_compiler_ver=2024.2.1 + +export stack_mpi=intel-oneapi-mpi +export stack_mpi_ver=2021.13 + +spack_stack_env="ue-${stack_compiler}-2024.1.0" +spack_stack_root="/apps/contrib/spack-stack" + source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" + +export intel_mkl_ver=2023.1.0 diff --git a/versions/build.noaacloud.ver b/versions/build.noaacloud.ver index b5fd272b4b5..a8f16cdfd4d 100644 --- a/versions/build.noaacloud.ver +++ b/versions/build.noaacloud.ver @@ -1,5 +1,13 @@ -export stack_intel_ver=2021.10.0 -export stack_impi_ver=2021.10.0 -export spack_env=gsi-addon-env +#!/usr/bin/env bash +export stack_compiler=oneapi +export stack_compiler_ver=2024.2.1 + +export stack_mpi=intel-oneapi-mpi +export stack_mpi_ver=2021.13 + +# gnu_ver set to override 14.2.0 default set in UFS-WM noaacloud module file +export gnu_ver=13.2.0 + +spack_stack_root="/contrib/spack-stack-rocky8" + source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/contrib/spack-stack-rocky8/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" diff --git a/versions/build.orion.ver b/versions/build.orion.ver index fde6136ad66..0ca97fa8cfc 100644 --- a/versions/build.orion.ver +++ b/versions/build.orion.ver @@ -1,5 +1,13 @@ -export stack_intel_ver=2021.9.0 -export stack_impi_ver=2021.9.0 -export spack_env=gsi-addon-dev-fms-2024.01 +#!/usr/bin/env bash +export stack_compiler=oneapi +export stack_compiler_ver=2024.2.1 + +export stack_mpi=intel-oneapi-mpi +export stack_mpi_ver=2021.13 + +spack_stack_env="ue-${stack_compiler}-2024.1.0" +spack_stack_root="/apps/contrib/spack-stack" + source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" + +export intel_mkl_ver=2023.1.0 diff --git a/versions/build.ursa.ver b/versions/build.ursa.ver new file mode 100644 index 00000000000..b35f8ee5d9a --- /dev/null +++ b/versions/build.ursa.ver @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +export stack_compiler=oneapi +export stack_compiler_ver=2024.2.1 + +export stack_mpi=intel-oneapi-mpi +export stack_mpi_ver=2021.13 + +spack_stack_root="/contrib/spack-stack" + +source "${HOMEgfs:-}/versions/spack.ver" diff --git a/versions/build.wcoss2.ver b/versions/build.wcoss2.ver index 3ae0b3a1cc4..dcebeac9829 100644 --- a/versions/build.wcoss2.ver +++ b/versions/build.wcoss2.ver @@ -1,32 +1,32 @@ +#!/usr/bin/env bash export PrgEnv_intel_ver=8.1.0 export intel_ver=19.1.3.304 export craype_ver=2.7.13 export cray_mpich_ver=8.1.9 - export cmake_ver=3.20.2 export jasper_ver=2.0.25 -export libpng_ver=1.6.37 export zlib_ver=1.2.11 -export esmf_ver=8.3.0b09 -export pio_ver=2.5.2 -export fms_ver=2022.04 +export libpng_ver=1.6.37 -export hdf5_ver=1.10.6 -export netcdf_ver=4.7.4 +export hdf5_ver=1.14.0 +export pnetcdf_ver=1.12.2 +export netcdf_ver=4.9.2 +export esmf_ver=8.8.0 +export pio_ver=2.5.2 +export fms_ver=2024.01 export bacio_ver=2.4.1 export w3nco_ver=2.4.1 export nemsio_ver=2.5.2 export sigio_ver=2.3.2 -export w3emc_ver=2.9.2 -export bufr_ver=11.7.0 -export g2_ver=3.4.5 -export sp_ver=2.3.3 -export ip_ver=3.3.3 +export w3emc_ver=2.12.0 +export bufr_ver=12.1.0 +export g2_ver=3.5.1 +export ip_ver=5.2.0 export wrf_io_ver=1.2.0 export ncio_ver=1.1.2 -export ncdiag_ver=1.0.0 +export ncdiag_ver=1.1.2 export g2tmpl_ver=1.10.2 export crtm_ver=2.4.0.1 diff --git a/versions/ic.ver b/versions/ic.ver index a632623d06c..b8c1be4da16 100644 --- a/versions/ic.ver +++ b/versions/ic.ver @@ -4,13 +4,15 @@ #shellcheck disable=SC2034 declare -A ic_versions -ic_versions['C48']=20250327 -ic_versions['C48mx500']=20250327 -ic_versions['C48C48mx500']=20250327 +ic_versions['C48']=20250808 +ic_versions['C48mx500']=20250808 +ic_versions['C48C48mx500']=20250808 -ic_versions['C96']=20250327 -ic_versions['C96C48']=20250327 -ic_versions['C96mx100']=20250327 +ic_versions['C96']=20250808 +ic_versions['C96C48']=20250808 + +# GEFS +ic_versions['C96mx100']=20240610 ic_versions['C384C192']=20240610 ic_versions['C384mx025']=20241120 diff --git a/versions/run.gaeac5.ver b/versions/run.gaeac5.ver deleted file mode 100644 index 6e61ca82295..00000000000 --- a/versions/run.gaeac5.ver +++ /dev/null @@ -1,11 +0,0 @@ -export stack_intel_ver=2023.2.0 -export stack_cray_mpich_ver=8.1.28 -export spack_env=gsi-addon-dev-fms-2024.01 - -export perl_ver=5.38.2 - -export hsi_mod_path="/usw/hpss/modulefiles" -export hsi_ver=9.3 - -source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.gaeac6.ver b/versions/run.gaeac6.ver index c342a313e65..68446906e38 100644 --- a/versions/run.gaeac6.ver +++ b/versions/run.gaeac6.ver @@ -1,12 +1,19 @@ -export stack_intel_ver=2023.2.0 -export stack_cray_mpich_ver=8.1.29 -export spack_env=gsi-addon-dev-fms-2024.01 +#!/usr/bin/env bash +export stack_compiler=intel +export stack_compiler_ver=2023.2.0 -export perl_ver=5.38.2 +export stack_mpi=cray-mpich +export stack_mpi_ver=8.1.30 + +spack_stack_root="/ncrc/proj/epic/spack-stack/c6" -export hsi_mod_path="/usw/hpss/modulefiles" -export hsi_ver=9.3 source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/ncrc/proj/epic/spack-stack/c6/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" +# Gaea uses a slgihtly older version of cdo +export cdo_ver=2.3.0 + +export perl_ver=5.38.2 + +export hsi_mod_path="/usw/hpss/modulefiles" +export hsi_ver=9.3 diff --git a/versions/run.hera.ver b/versions/run.hera.ver index 5df640bfec6..1e228c55092 100644 --- a/versions/run.hera.ver +++ b/versions/run.hera.ver @@ -1,6 +1,16 @@ -export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-dev-fms-2024.01 +#!/usr/bin/env bash +export stack_compiler=oneapi +export stack_compiler_ver=2024.2.1 + +export stack_mpi=intel-oneapi-mpi +export stack_mpi_ver=2021.13 + +spack_stack_root="/contrib/spack-stack" + +source "${HOMEgfs:-}/versions/spack.ver" + +# Hera has an older version of CDO installed +export cdo_ver=2.3.0 export hpss_ver=hpss export ncl_ver=6.6.2 @@ -8,8 +18,4 @@ export R_ver=3.6.1 export gempak_ver=7.17.0 export perl_ver=5.38.0 - -source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" - export imagemagick_ver=7.1.1-11 diff --git a/versions/run.hercules.ver b/versions/run.hercules.ver index 7ba02511335..7b5ad7ea3c6 100644 --- a/versions/run.hercules.ver +++ b/versions/run.hercules.ver @@ -1,8 +1,14 @@ -export stack_intel_ver=2021.9.0 -export stack_impi_ver=2021.9.0 -export intel_mkl_ver=2023.1.0 -export spack_env=gsi-addon-dev-fms-2024.01 -export globus_cli_ver=3.27 +#!/usr/bin/env bash +export stack_compiler=oneapi +export stack_compiler_ver=2024.2.1 + +export stack_mpi=intel-oneapi-mpi +export stack_mpi_ver=2021.13 + +spack_stack_env="ue-${stack_compiler}-2024.1.0" +spack_stack_root="/apps/contrib/spack-stack" + source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" + export sven_root_path="/home/gfekete/sven" +export intel_mkl_ver=2023.1.0 diff --git a/versions/run.noaacloud.ver b/versions/run.noaacloud.ver index 88fe4e0115f..b8621a10f57 100644 --- a/versions/run.noaacloud.ver +++ b/versions/run.noaacloud.ver @@ -1,14 +1,15 @@ -export stack_intel_ver=2021.10.0 -export stack_impi_ver=2021.10.0 -export spack_env=gsi-addon-env +#!/usr/bin/env bash +export stack_compiler=oneapi +export stack_compiler_ver=2024.2.1 -source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/contrib/spack-stack-rocky8/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" +export stack_mpi=intel-oneapi-mpi +export stack_mpi_ver=2021.13 -export cdo_ver=2.2.0 +spack_stack_root="/contrib/spack-stack-rocky8" -# modules updated in spack.ver for gsi-addon-dev-fms-2024.01 -export export esmf_ver=8.5.0 -export g2tmpl_ver=1.10.2 +source "${HOMEgfs:-}/versions/spack.ver" +export cdo_ver=2.3.0 export perl_ver=5.38.0 +export mkl_ver=2024.2.1 +export imagemagick_ver=7.1.1-11 diff --git a/versions/run.orion.ver b/versions/run.orion.ver index 4cccb09b7e1..7b5ad7ea3c6 100644 --- a/versions/run.orion.ver +++ b/versions/run.orion.ver @@ -1,8 +1,14 @@ -export stack_intel_ver=2021.9.0 -export stack_impi_ver=2021.9.0 -export intel_mkl_ver=2023.1.0 -export spack_env=gsi-addon-dev-fms-2024.01 -export globus_cli_ver=3.27 +#!/usr/bin/env bash +export stack_compiler=oneapi +export stack_compiler_ver=2024.2.1 + +export stack_mpi=intel-oneapi-mpi +export stack_mpi_ver=2021.13 + +spack_stack_env="ue-${stack_compiler}-2024.1.0" +spack_stack_root="/apps/contrib/spack-stack" + source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" -export sven_root_path="/home/dhuber/sven" + +export sven_root_path="/home/gfekete/sven" +export intel_mkl_ver=2023.1.0 diff --git a/versions/run.ursa.ver b/versions/run.ursa.ver new file mode 100644 index 00000000000..9b7d3c9502c --- /dev/null +++ b/versions/run.ursa.ver @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +export stack_compiler=oneapi +export stack_compiler_ver=2024.2.1 + +export stack_mpi=intel-oneapi-mpi +export stack_mpi_ver=2021.13 + +spack_stack_root="/contrib/spack-stack" + +source "${HOMEgfs:-}/versions/spack.ver" + +export imagemagick_ver=7.1.1-29 diff --git a/versions/run.wcoss2.ver b/versions/run.wcoss2.ver index d67f14bc9c4..e030c95a3a6 100644 --- a/versions/run.wcoss2.ver +++ b/versions/run.wcoss2.ver @@ -1,3 +1,4 @@ +#! /usr/bin/env bash export version=v16.3.7 export gfs_ver=v16.3.7 export ukmet_ver=v2.2 @@ -5,6 +6,15 @@ export ecmwf_ver=v2.1 export nam_ver=v4.2 export rtofs_ver=v2.3 export obsproc_ver=v1.1 +export imagemagick_ver=7.0.8-7 +export cdo_ver=1.9.8 +export perl_ver=5.32.0 +export python_ver=3.8.6 +export gempak_ver=7.15.1 +export nco_ver=5.0.6 +export grib_util_ver=1.2.3 +export bufr_dump_ver=1.0.0 +export wgrib2_ver=2.0.8 export envvar_ver=1.0 export prod_envir_ver=${prod_envir_ver:-2.0.4} # Allow override from ops ecflow @@ -17,36 +27,29 @@ export cray_mpich_ver=8.1.9 export cray_pals_ver=1.0.17 export cfp_ver=2.0.4 -export python_ver=3.8.6 -export gempak_ver=7.15.1 -export perl_ver=5.32.0 export libjpeg_ver=9c export udunits_ver=2.2.28 export gsl_ver=2.7 export jasper_ver=2.0.25 export zlib_ver=1.2.11 export libpng_ver=1.6.37 -export cdo_ver=1.9.8 -export imagemagick_ver=7.0.8-7 -export hdf5_ver=1.10.6 -export netcdf_ver=4.7.4 +export hdf5_ver=1.14.0 +export pnetcdf_ver=1.12.2 +export netcdf_ver=4.9.2 +export esmf_ver=8.8.0 -export nco_ver=4.7.9 -export grib_util_ver=1.2.3 -export bufr_dump_ver=1.0.0 export util_shared_ver=1.4.0 +export ncdiag_ver=1.1.2 export g2tmpl_ver=1.10.2 -export ncdiag_ver=1.0.0 export crtm_ver=2.4.0.1 -export wgrib2_ver=2.0.8 export met_ver=9.1.3 export metplus_ver=3.1.1 # Development-only below export obsproc_run_ver=1.2.0 -export prepobs_run_ver=1.1.0 +export prepobs_run_ver=1.1.2 -export ens_tracker_ver=v1.2.0 -export fit2obs_ver=1.1.7.1 +export ens_tracker_ver=v1.2.1 +export fit2obs_ver=1.1.9 diff --git a/versions/spack.ver b/versions/spack.ver index 7d27b3e9a03..d763160bf5e 100644 --- a/versions/spack.ver +++ b/versions/spack.ver @@ -1,47 +1,52 @@ -export spack_stack_ver=1.6.0 -export cmake_ver=3.23.1 -export python_ver=3.11.6 +#! /usr/bin/env bash +spack_stack_env="${spack_stack_env:-ue-${stack_compiler}-${stack_compiler_ver}}" +spack_stack_ver="${spack_stack_ver:-1.9.2}" +export stack_core_path="${spack_stack_root}/spack-stack-${spack_stack_ver}/envs/${spack_stack_env}/install/modulefiles/Core" +export cmake_ver=3.27.9 +export stack_python_ver=3.11.7 export jasper_ver=2.0.32 export libpng_ver=1.6.37 export zlib_ver=1.2.13 -export esmf_ver=8.6.0 -export fms_ver=2023.02.01 -export cdo_ver=2.2.0 -export nco_ver=5.0.6 +export esmf_ver=8.8.0 +export fms_ver=2024.02 +export cdo_ver=2.4.4 +export nco_ver=5.2.4 -export hdf5_ver=1.14.0 +export hdf5_ver=1.14.3 export netcdf_c_ver=4.9.2 export netcdf_fortran_ver=4.6.1 export bacio_ver=2.4.1 export nemsio_ver=2.5.4 -export sigio_ver=2.3.2 +export sigio_ver=2.3.3 export w3emc_ver=2.10.0 -export bufr_ver=11.7.0 -export g2_ver=3.4.5 +export bufr_ver=12.1.0 +export g2_ver=3.5.1 export sp_ver=2.5.0 -export ip_ver=4.3.0 +export ip_ver=5.1.0 export gsi_ncdiag_ver=1.1.2 export g2tmpl_ver=1.13.0 export crtm_ver=2.4.0.1 -export wgrib2_ver=2.0.8 -export grib_util_ver=1.3.0 +export wgrib2_ver=3.6.0 +export grib_util_ver=1.4.0 export prod_util_ver=2.1.1 -export py_netcdf4_ver=1.5.8 -export py_pyyaml_ver=6.0 -export py_jinja2_ver=3.1.2 -export py_pandas_ver=1.5.3 +export py_netcdf4_ver=1.7.1.post2 +export py_pyyaml_ver=6.0.2 +export py_jinja2_ver=3.1.4 +export py_pandas_ver=2.2.3 export py_python_dateutil_ver=2.8.2 export py_f90nml_ver=1.4.3 export py_numpy_ver=1.23.4 -export met_ver=9.1.3 -export metplus_ver=3.1.1 -export py_xarray_ver=2023.7.0 +export py_xarray_ver=2024.7.0 + +# NOTE These versions are incompatible with verif-global +export met_ver=11.1.1 +export metplus_ver=5.1.0 export obsproc_run_ver=1.2.0 -export prepobs_run_ver=1.1.0 +export prepobs_run_ver=1.1.2 -export ens_tracker_ver=v1.2.0 -export fit2obs_ver=1.1.7.1 +export ens_tracker_ver=v1.2.1 +export fit2obs_ver=1.1.9