diff --git a/README.md b/README.md index 3311f540a5c..135e3a465d8 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,6 @@ The Global Workflow currently supports the following machines at the indicated t | Orion
MSU | 2 | The GSI runs very slowly on Orion and the TC tracker is not supported. | | Gaea C5/C6
RDHPCS | 3 | Currently non-operational following an OS upgrade.
Supported by EPIC. | | AWS, GCP, Azure
NOAA Parallel Works | 3 | Supported by EPIC. | -| Jet
RDHPCS | 3 | Supported by NESDIS. | -| S4
SSEC | 3 | Currently non-operational following an OS upgrade.
Supported by NESDIS. | **Tier Definitions** diff --git a/dev/workflow/hosts.py b/dev/workflow/hosts.py index 834027beedc..70a337ff63d 100644 --- a/dev/workflow/hosts.py +++ b/dev/workflow/hosts.py @@ -15,9 +15,8 @@ class Host: Gather Host specific information. """ - SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', 'HERCULES', - 'WCOSS2', 'S4', 'CONTAINER', 'GAEAC5', - 'GAEAC6', 'AWSPW', 'AZUREPW', 'GOOGLEPW'] + SUPPORTED_HOSTS = ['HERA', 'ORION', 'HERCULES', 'WCOSS2', 'CONTAINER', + 'GAEAC5', 'GAEAC6', 'AWSPW', 'AZUREPW', 'GOOGLEPW'] def __init__(self, host=None): @@ -56,12 +55,8 @@ def detect(cls): machine = 'HERA' elif os.path.exists('/work/noaa'): machine = socket.gethostname().split("-", 1)[0].upper() - elif os.path.exists('/lfs5/HFIP'): - machine = 'JET' elif os.path.exists('/lfs/f1'): machine = 'WCOSS2' - elif os.path.exists('/data/prod'): - machine = 'S4' elif os.path.exists('/gpfs/f5'): machine = 'GAEAC5' elif os.path.exists('/gpfs/f6'): diff --git a/dev/workflow/hosts/jet.yaml b/dev/workflow/hosts/jet.yaml deleted file mode 100644 index 34ed0f44936..00000000000 --- a/dev/workflow/hosts/jet.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Paths -DMPDIR: '/lfs5/HFIP/hfv3gfs/glopara/dump' -BASE_GIT: '/lfs5/HFIP/hfv3gfs/glopara/git' -BASE_DATA: '/lfs5/HFIP/hfv3gfs/glopara/data' -BASE_IC: '/mnt/lfs5/HFIP/hfv3gfs/glopara/data/ICSDIR' -AERO_INPUTS_DIR: /lfs5/HFIP/hfv3gfs/glopara/data/GEFS_ExtData/20250310 -PACKAGEROOT: '/lfs5/HFIP/hfv3gfs/glopara/nwpara' -HOMEDIR: '/lfs5/HFIP/hfv3gfs/${USER}' -STMP: '/lfs5/HFIP/hfv3gfs/${USER}/stmp' -PTMP: '/lfs5/HFIP/hfv3gfs/${USER}/ptmp' -NOSCRUB: '${HOMEDIR}' -COMINsyn: '/lfs5/HFIP/hfv3gfs/glopara/com/gfs/prod/syndat' -COMINecmwf: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/ecmwf -COMINnam: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/nam -COMINukmet: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/ukmet -# BQS properties -SCHEDULER: slurm -QUEUE: batch -PARTITION_BATCH: kjet -PARTITION_SERVICE: service -# HPSS properties -HPSS_PROJECT: emc-global -ARCHCOM_TO: 'hpss' -ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' -CHGRP_RSTPROD: 'YES' -CHGRP_CMD: 'chgrp rstprod' -# Features -SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] -DO_ARCHCOM: 'YES' -DO_AWIPS: 'NO' -MAKE_NSSTBUFR: 'NO' -MAKE_ACFTBUFR: 'NO' diff --git a/dev/workflow/hosts/s4.yaml b/dev/workflow/hosts/s4.yaml deleted file mode 100644 index bc8c9ab1a31..00000000000 --- a/dev/workflow/hosts/s4.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Paths -DMPDIR: '/data/prod/glopara/dump' -BASE_GIT: '/data/prod/glopara/git' -BASE_DATA: '/data/prod/glopara' -BASE_IC: '/data/prod/glopara/coupled_ICs' -AERO_INPUTS_DIR: /data/prod/glopara/GEFS_ExtData/20250310 -PACKAGEROOT: '/data/prod/glopara/nwpara' -HOMEDIR: '/data/users/${USER}' -STMP: '/scratch/users/${USER}' -PTMP: '/scratch/users/${USER}' -NOSCRUB: '${HOMEDIR}' -COMINsyn: '/data/prod/glopara/com/gfs/prod/syndat' -# BQS properties -SCHEDULER: slurm -QUEUE: s4 -QUEUE_SERVICE: serial -PARTITION_BATCH: s4 -PARTITION_SERVICE: serial -# HPSS properties -HPSS_PROJECT: emc-global -ARCHCOM_TO: 'local' -ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' -CHGRP_RSTPROD: 'NO' -CHGRP_CMD: 'ls' -# Features -SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] -DO_ARCHCOM: 'NO' -MAKE_NSSTBUFR: 'YES' -MAKE_ACFTBUFR: 'YES' diff --git a/docs/doxygen/compile b/docs/doxygen/compile index cc43994ccb7..c40a21d723c 100755 --- a/docs/doxygen/compile +++ b/docs/doxygen/compile @@ -9,8 +9,6 @@ if [ $machine = "HERA" ]; then doxygen=/usr/bin/doxygen elif [ $machine = "ORION" ]; then doxygen=/bin/doxygen -elif [ $machine = "JET" ]; then - doxygen=/contrib/doxygen/1.8.10/bin/doxygen else echo "machine $machine is unrecognized!" echo "trying system doxygen" diff --git a/docs/note_fixfield.txt b/docs/note_fixfield.txt deleted file mode 100644 index 32e44b9d98b..00000000000 --- a/docs/note_fixfield.txt +++ /dev/null @@ -1,16 +0,0 @@ ------------------------------------------------------------------------------- -For EMC, the fix fields for running the model are not included in git repository. -They are saved locally on all platforms - -Hera: /scratch1/NCEPDEV/global/glopara/fix -Orion/Hercules: /work/noaa/global/glopara/fix -Jet: /mnt/lfs5/HFIP/hfv3gfs/glopara/FIX/fix -S4: /data/prod/glopara/fix - ------------------------------------------------------------------------------- -09/28/2018 - -In fix/fix_am -rename current syndat_stmnames to syndat_stmnames_q2fy19 (used for FV3GFS Q2FY19 parallels) -copy /gpfs/hps/nco/ops/nwprod/global_shared.v14.1.6/fix/syndat_stmnames ./fix/fix_am/syndat_stmnames - diff --git a/docs/source/components.rst b/docs/source/components.rst index 98803c96c7c..2a0727949c4 100644 --- a/docs/source/components.rst +++ b/docs/source/components.rst @@ -60,9 +60,7 @@ Observation data, also known as dump data, is prepared in production and then ar * Hera: ``/scratch1/NCEPDEV/global/glopara/dump`` * Orion/Hercules: ``/work/noaa/rstprod/dump`` -* Jet: ``/mnt/lfs5/HFIP/hfv3gfs/glopara/dump`` * WCOSS2: ``/lfs/h2/emc/global/noscrub/emc.global/dump`` -* S4: ``/data/prod/glopara/dump`` ----------------------------- Global Dump Archive Structure diff --git a/docs/source/hpc.rst b/docs/source/hpc.rst index a74af946726..437147ed74e 100644 --- a/docs/source/hpc.rst +++ b/docs/source/hpc.rst @@ -202,34 +202,6 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas - - - - * - Jet - - 3 - - X - - - - - - X - - - - - - X - - - - - - X - - X - - X - * - S4 - - 3 - - - - - - - - X - - - - - - X - - - - - - X - - X - - ================================ Experiment troubleshooting help @@ -249,8 +221,6 @@ HPC helpdesks * Hercules: rdhpcs.hercules.help@noaa.gov * HPSS: rdhpcs.hpss.help@noaa.gov * Gaea: oar.gfdl.help@noaa.gov -* S4: innocent.souopgui@noaa.gov -* Jet: rdhpcs.jet.help@noaa.gov * Cloud: rdhpcs.cloud.help@noaa.gov ====================== @@ -259,8 +229,6 @@ Restricted data access The GFS system ingests dump data files that contain global observation data. A number of these dump files contain restricted data which means those files come with an extra level of permissions called restricted or ‘rstprod’. Users who wish to run cycled GFS experiments, which both utilizes restricted observation data and produces output containing restricted data, will need to gain rstprod group access. -NOTE: Only non-restricted data is available on S4. - To request rstprod access, do either a and/or b below: a) If you need restricted data access on WCOSS2, read details about restricted data and fill out form here: @@ -271,15 +239,9 @@ b) If you need restricted data access on RDHPCS systems: go to the AIM system, c https://aim.rdhpcs.noaa.gov/ -==================================== -Optimizing the global workflow on S4 -==================================== - -The S4 cluster is relatively small and so optimizations are recommended to improve cycled runtimes. Please contact Innocent Souopgui (innocent.souopgui@noaa.gov) if you are planning on running a cycled experiment on this system to obtain optimized configuration files. - -================================================== -Stacksize on R&Ds (Hera, Orion, Hercules, Jet, S4) -================================================== +========================================= +Stacksize on R&Ds (Hera, Orion, Hercules) +========================================= Some GFS components, like the UPP, need an unlimited stacksize. Add the following setting into your appropriate .*rc file to support these components: diff --git a/docs/source/init.rst b/docs/source/init.rst index bd1bc1d2ce0..2f119c6a235 100644 --- a/docs/source/init.rst +++ b/docs/source/init.rst @@ -111,7 +111,6 @@ Warm-start cycled w/ coupled (S2S) model C48 atmosphere C48 enkf (80 members) 5 Hera: /scratch1/NCEPDEV/global/glopara/data/ICSDIR/C48C48mx500 Orion/Hercules: /work/noaa/global/glopara/data/ICSDIR/C48C48mx500 WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR/C48C48mx500 - Jet: /lfs5/HFIP/hfv3gfs/glopara/data/ICSDIR/C48C48mx500 AWS: https://noaa-nws-global-pds.s3.amazonaws.com/index.html#data/ICSDIR/C48C48mx500 Start date = 2021032312 @@ -227,8 +226,6 @@ Forecast-only P8 prototype initial conditions are made available to users on sup WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/IC/COUPLED HERA: /scratch1/NCEPDEV/climate/role.ufscpara/IC ORION/Hercules: /work/noaa/global/glopara/data/ICSDIR/prototype_ICs - JET: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/ICSDIR/prototype_ICs - S4: /data/prod/glopara/coupled_ICs These locations are known within the workflow via paths set in ``parm/config/config.coupled_ic``. @@ -255,7 +252,7 @@ Not yet supported. See the UFS_UTILS documentation on the gdas_init utility to g --------------------- Forecast-only coupled --------------------- -Coupled initial conditions are currently only generated offline and copied prior to the forecast run. Prototype initial conditions will automatically be used when setting up an experiment as an S2SW app, there is no need to do anything additional. Sample copies of initial conditions from the prototype runs are currently maintained on Hera, Orion/Hercules, Jet, and WCOSS2. The locations used are determined by ``parm/config/config.stage_ic``. +Coupled initial conditions are currently only generated offline and copied prior to the forecast run. Prototype initial conditions will automatically be used when setting up an experiment as an S2SW app, there is no need to do anything additional. Sample copies of initial conditions from the prototype runs are currently maintained on Hera, Orion/Hercules, and WCOSS2. The locations used are determined by ``parm/config/config.stage_ic``. Note however, that due to the rapid changes in the model configuration, some staged initial conditions may not work. .. _forecastonly-atmonly: @@ -314,9 +311,6 @@ For HPSS path, see retrospective table in :ref:`pre-production parallel section Manual Generation ***************** -.. note:: - Initial conditions cannot be generated on S4. These must be generated on another supported platform then pushed to S4. If you do not have access to a supported system or need assistance, please contact Innocent Souopgui (innocent.souopgui@noaa.gov). - .. _coldstarts: The following information is for users needing to generate cold-start initial conditions for a cycled experiment that will run at a different resolution or layer amount than the operational GFS (C768C384L127). diff --git a/env/JET.env b/env/JET.env deleted file mode 100755 index 70ca192c0fe..00000000000 --- a/env/JET.env +++ /dev/null @@ -1,297 +0,0 @@ -#! /usr/bin/env bash - -if [[ $# -ne 1 ]]; then - - echo "Must specify an input argument to set runtime environment variables!" - exit 1 - -fi - -step=$1 - -export launcher="srun -l --epilog=/apps/local/bin/report-mem --export=ALL" -export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" - -# Configure MPI environment -export OMP_STACKSIZE=2048000 -export NTHSTACK=1024000000 - -ulimit -s unlimited -ulimit -a - -# Calculate common variables -# Check first if the dependent variables are set -if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then - max_threads_per_task=$((max_tasks_per_node / tasks_per_node)) - NTHREADSmax=${threads_per_task:-${max_threads_per_task}} - NTHREADS1=${threads_per_task:-1} - if [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]]; then - NTHREADSmax=${max_threads_per_task} - fi - if [[ ${NTHREADS1} -gt ${max_threads_per_task} ]]; then - NTHREADS1=${max_threads_per_task} - fi - APRUN_default="${launcher} -n ${ntasks}" -else - echo "ERROR config.resources must be sourced before sourcing JET.env" - exit 2 -fi - -if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then - - export POE="NO" - export BACK="NO" - export sys_tp="JET" - export launcher_PREP="srun" - -elif [[ "${step}" = "prep_emissions" ]]; then - - export APRUN="${launcher} -n 1" - -elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then - - export USE_CFP="YES" - if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi - export wavempexec=${launcher} - export wave_mpmd=${mpmd_opt} - -elif [[ "${step}" = "atmanlvar" ]]; then - - export NTHREADS_ATMANLVAR=${NTHREADSmax} - export APRUN_ATMANLVAR="${APRUN_default}" - -elif [[ "${step}" = "atmensanlobs" ]]; then - - export NTHREADS_ATMENSANLOBS=${NTHREADSmax} - export APRUN_ATMENSANLOBS="${APRUN_default}" - -elif [[ "${step}" = "atmensanlsol" ]]; then - - export NTHREADS_ATMENSANLSOL=${NTHREADSmax} - export APRUN_ATMENSANLSOL="${APRUN_default}" - -elif [[ "${step}" = "atmensanlletkf" ]]; then - - export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} - export APRUN_ATMENSANLLETKF="${launcher} ${ntasks}" - -elif [[ "${step}" = "atmensanlfv3inc" ]]; then - - export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} - export APRUN_ATMENSANLFV3INC="${launcher} ${ntasks}" - -elif [[ "${step}" = "aeroanlvar" ]]; then - - export NTHREADS_AEROANL=${NTHREADSmax} - export APRUN_AEROANL="${APRUN_default}" - -elif [[ "${step}" = "aeroanlgenb" ]]; then - - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - - export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANLGENB}" - -elif [[ "${step}" = "prepobsaero" ]]; then - - export NTHREADS_PREPOBSAERO=${NTHREADS1} - export APRUN_PREPOBSAERO="${APRUN_default} --cpus-per-task=${NTHREADS_PREPOBSAERO}" - -elif [[ "${step}" = "snowanl" ]]; then - - export APRUN_CALCFIMS="${launcher} -n 1" - - export NTHREADS_SNOWANL=${NTHREADSmax} - export APRUN_SNOWANL="${APRUN_default}" - - export APRUN_APPLY_INCR="${launcher} -n 6" - -elif [[ "${step}" = "esnowanl" ]]; then - - export APRUN_CALCFIMS="${launcher} -n 1" - - export NTHREADS_ESNOWANL=${NTHREADSmax} - export APRUN_ESNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWANL}" - - export APRUN_APPLY_INCR="${launcher} -n 6" - -elif [[ "${step}" = "atmanlfv3inc" ]]; then - - export NTHREADS_ATMANLFV3INC=${NTHREADSmax} - export APRUN_ATMANLFV3INC="${APRUN_default}" - -elif [[ "${step}" = "marinebmat" ]]; then - - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - export APRUN_MARINEBMAT="${APRUN_default}" - -elif [[ "${step}" = "marineanlvar" ]]; then - - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - export APRUN_MARINEANLVAR="${APRUN_default}" - -elif [[ "${step}" = "ecen_fv3jedi" ]]; then - - export NTHREADS_ECEN_FV3JEDI=${NTHREADSmax} - export APRUN_CORRECTION_INCREMENT="${launcher} -n ${ntasks_correction_increment} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}" - export APRUN_ENSEMBLE_RECENTER="${launcher} -n ${ntasks_ensemble_recenter} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}" - -elif [[ "${step}" = "analcalc_fv3jedi" ]]; then - - export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax} - export APRUN_ANALCALC_FV3JEDI="${APRUN_default}" - -elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then - - export MKL_NUM_THREADS=4 - export MKL_CBWR=AUTO - - export CFP_MP=${CFP_MP:-"YES"} - export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - - export NTHREADS_GSI=${threads_per_task_anal:-${max_threads_per_task}} - export APRUN_GSI="${APRUN_default}" - - export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} - if [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]]; then - export NTHREADS_CALCINC=${max_threads_per_task} - fi - export APRUN_CALCINC="${launcher} \$ncmd" - - export NTHREADS_CYCLE=${threads_per_task_cycle:-12} - if [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]]; then - export NTHREADS_CYCLE=${max_tasks_per_node} - fi - ntasks_cycle=${ntiles:-6} - export APRUN_CYCLE="${launcher} -n ${ntasks_cycle}" - - export NTHREADS_GAUSFCANL=1 - ntasks_gausfcanl=${ntasks_gausfcanl:-1} - export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl}" - -elif [[ "${step}" = "sfcanl" ]]; then - export NTHREADS_CYCLE=${threads_per_task:-14} - if [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]]; then - export NTHREADS_CYCLE=${max_tasks_per_node} - fi - export APRUN_CYCLE="${APRUN_default}" - - # REGRID requires multiple of 6 tasks - ntdiv6=$((ntasks/6)) - ntasks_regrid=$((ntdiv6*6)) - if [[ ${ntasks_regrid} -gt 36 ]]; then - ntasks_regrid=36 - fi - export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " - -elif [[ "${step}" = "eobs" ]]; then - - export MKL_NUM_THREADS=4 - export MKL_CBWR=AUTO - - export NTHREADS_GSI=${NTHREADSmax} - export APRUN_GSI="${APRUN_default}" - - export CFP_MP=${CFP_MP:-"YES"} - export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - -elif [[ "${step}" = "eupd" ]]; then - - export NTHREADS_ENKF=${NTHREADSmax} - export APRUN_ENKF="${launcher} -n ${ntasks_enkf:-${ntasks}}" - - export CFP_MP=${CFP_MP:-"YES"} - export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - -elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then - - (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) - (( ufs_ntasks = nnodes*tasks_per_node )) - # With ESMF threading, the model wants to use the full node - export APRUN_UFS="${launcher} -n ${ufs_ntasks}" - unset nnodes ufs_ntasks - -elif [[ "${step}" = "upp" ]]; then - - export NTHREADS_UPP=${NTHREADS1} - export APRUN_UPP="${APRUN_default}" - -elif [[ "${step}" = "atmos_products" ]]; then - - export USE_CFP="YES" # Use MPMD for downstream product generation - -elif [[ "${step}" = "oceanice_products" ]]; then - - export NTHREADS_OCNICEPOST=${NTHREADS1} - export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}" - -elif [[ "${step}" = "ecen" ]]; then - - export NTHREADS_ECEN=${NTHREADSmax} - export APRUN_ECEN="${APRUN_default}" - - export NTHREADS_CHGRES=${threads_per_task_chgres:-12} - if [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]]; then - export NTHREADS_CHGRES=${max_tasks_per_node} - fi - export APRUN_CHGRES="time" - - export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} - if [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]]; then - export NTHREADS_CALCINC=${max_threads_per_task} - fi - export APRUN_CALCINC="${APRUN_default}" - -elif [[ "${step}" = "esfc" ]]; then - - export NTHREADS_ESFC=${NTHREADSmax} - export APRUN_ESFC="${APRUN_default}" - - export NTHREADS_CYCLE=${threads_per_task_cycle:-14} - if [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]]; then - export NTHREADS_CYCLE=${max_tasks_per_node} - fi - export APRUN_CYCLE="${APRUN_default}" - - ntdiv6=$((ntasks/6)) - ntasks_regrid=$((ntdiv6*6)) - if [[ ${ntasks_regrid} -gt 36 ]]; then - ntasks_regrid=36 - fi - export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " - -elif [[ "${step}" = "epos" ]]; then - - export NTHREADS_EPOS=${NTHREADSmax} - export APRUN_EPOS="${APRUN_default}" - -elif [[ "${step}" = "postsnd" ]]; then - - export CFP_MP="YES" - - export NTHREADS_POSTSND=${NTHREADS1} - export APRUN_POSTSND="${APRUN_default}" - - export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1} - if [[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]]; then - export NTHREADS_POSTSNDCFP=${max_threads_per_task} - fi - export APRUN_POSTSNDCFP="${launcher} -n ${ntasks_postsndcfp} ${mpmd_opt}" - -elif [[ "${step}" = "awips" ]]; then - - echo "WARNING: ${step} is not enabled on ${machine}!" - -elif [[ "${step}" = "gempak" ]]; then - - echo "WARNING: ${step} is not enabled on ${machine}!" - -elif [[ "${step}" = "fit2obs" ]]; then - - export NTHREADS_FIT2OBS=${NTHREADS1} - export MPIRUN="${APRUN_default}" - -fi diff --git a/env/S4.env b/env/S4.env deleted file mode 100755 index 020650c4fe9..00000000000 --- a/env/S4.env +++ /dev/null @@ -1,259 +0,0 @@ -#! /usr/bin/env bash - -if [[ $# -ne 1 ]]; then - - echo "Must specify an input argument to set runtime environment variables!" - exit 1 - -fi - -step=$1 - -export launcher="srun -l --export=ALL" -export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" - -# Configure MPI environment -export OMP_STACKSIZE=2048000 -export NTHSTACK=1024000000 - -ulimit -s unlimited -ulimit -a - -# Calculate common variables -# Check first if the dependent variables are set -if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then - max_threads_per_task=$((max_tasks_per_node / tasks_per_node)) - NTHREADSmax=${threads_per_task:-${max_threads_per_task}} - NTHREADS1=${threads_per_task:-1} - if [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]]; then - NTHREADSmax=${max_threads_per_task} - fi - if [[ ${NTHREADS1} -gt ${max_threads_per_task} ]]; then - NTHREADS1=${max_threads_per_task} - fi - APRUN_default="${launcher} -n ${ntasks}" -else - echo "ERROR config.resources must be sourced before sourcing S4.env" - exit 2 -fi - -if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then - - export POE="NO" - export BACK="NO" - export sys_tp="S4" - export launcher_PREP="srun" - -elif [[ "${step}" = "prep_emissions" ]]; then - - export APRUN="${APRUN_default}" - -elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then - - export USE_CFP="YES" - if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi - export wavempexec=${launcher} - export wave_mpmd=${mpmd_opt} - -elif [[ "${step}" = "atmanlvar" ]]; then - - export NTHREADS_ATMANLVAR=${NTHREADSmax} - export APRUN_ATMANLVAR="${APRUN_default}" - -elif [[ "${step}" = "atmensanlobs" ]]; then - - export NTHREADS_ATMENSANLOBS=${NTHREADSmax} - export APRUN_ATMENSANLOBS="${APRUN_default}" - -elif [[ "${step}" = "atmensanlsol" ]]; then - - export NTHREADS_ATMENSANLSOL=${NTHREADSmax} - export APRUN_ATMENSANLSOL="${APRUN_default}" - -elif [[ "${step}" = "atmensanlletkf" ]]; then - - export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} - export APRUN_ATMENSANLLETKF="${APRUN_default}" - -elif [[ "${step}" = "atmensanlfv3inc" ]]; then - - export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} - export APRUN_ATMENSANLFV3INC="${APRUN_default}" - -elif [[ "${step}" = "aeroanlvar" ]]; then - - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - - export NTHREADS_AEROANL=${NTHREADSmax} - export APRUN_AEROANL="${APRUN_default}" - -elif [[ "${step}" = "aeroanlgenb" ]]; then - - export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANLGENB}" - -elif [[ "${step}" = "prepobsaero" ]]; then - - export NTHREADS_PREPOBSAERO=${NTHREADS1} - export APRUN_PREPOBSAERO="${APRUN_default} --cpus-per-task=${NTHREADS_PREPOBSAERO}" - -elif [[ "${step}" = "snowanl" ]]; then - - export APRUN_CALCFIMS="${launcher} -n 1" - - export NTHREADS_SNOWANL=${NTHREADSmax} - export APRUN_SNOWANL="${APRUN_default}" - - export APRUN_APPLY_INCR="${launcher} -n 6" - -elif [[ "${step}" = "esnowanl" ]]; then - - export APRUN_CALCFIMS="${launcher} -n 1" - - export NTHREADS_ESNOWANL=${NTHREADSmax} - export APRUN_ESNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWANL}" - - export APRUN_APPLY_INCR="${launcher} -n 6" - -elif [[ "${step}" = "atmanlfv3inc" ]]; then - - export NTHREADS_ATMANLFV3INC=${NTHREADSmax} - export APRUN_ATMANLFV3INC="${APRUN_default}" - -elif [[ "${step}" = "marinebmat" ]]; then - echo "WARNING: ${step} is not enabled on S4!" - -elif [[ "${step}" = "marinerun" ]]; then - echo "WARNING: ${step} is not enabled on S4!" - -elif [[ "${step}" = "ecen_fv3jedi" ]]; then - - export NTHREADS_ECEN_FV3JEDI=${NTHREADSmax} - export APRUN_CORRECTION_INCREMENT="${launcher} -n ${ntasks_correction_increment} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}" - export APRUN_ENSEMBLE_RECENTER="${launcher} -n ${ntasks_ensemble_recenter} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}" - -elif [[ "${step}" = "analcalc_fv3jedi" ]]; then - - export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax} - export APRUN_ANALCALC_FV3JEDI="${APRUN_default}" - -elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then - - export MKL_NUM_THREADS=4 - export MKL_CBWR=AUTO - - export CFP_MP=${CFP_MP:-"YES"} - export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - - export NTHREADS_GSI=${NTHREADSmax} - export APRUN_GSI="${APRUN_default}" - - export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} - if [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]]; then - export NTHREADS_CALCINC=${max_threads_per_task} - fi - export APRUN_CALCINC="${launcher} \$ncmd" - - export NTHREADS_CYCLE=${threads_per_task_cycle:-12} - if [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]]; then - export NTHREADS_CYCLE=${max_tasks_per_node} - fi - ntasks_cycle=${ntiles:-6} - export APRUN_CYCLE="${launcher} -n ${ntasks_cycle}" - - - export NTHREADS_GAUSFCANL=1 - ntasks_gausfcanl=${ntasks_gausfcanl:-1} - export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl}" - -elif [[ "${step}" = "sfcanl" ]]; then - export NTHREADS_CYCLE=${threads_per_task:-14} - if [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]]; then - export NTHREADS_CYCLE=${max_tasks_per_node} - fi - export APRUN_CYCLE="${APRUN_default}" - -elif [[ "${step}" = "eobs" ]]; then - - export MKL_NUM_THREADS=4 - export MKL_CBWR=AUTO - - export NTHREADS_GSI=${NTHREADSmax} - export APRUN_GSI="${APRUN_default}" - - export CFP_MP=${CFP_MP:-"YES"} - export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - -elif [[ "${step}" = "eupd" ]]; then - - export NTHREADS_ENKF=${NTHREADSmax} - export APRUN_ENKF="${launcher} -n ${ntasks_enkf:-${ntasks}}" - - export CFP_MP=${CFP_MP:-"YES"} - export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - -elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then - - (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) - (( ufs_ntasks = nnodes*tasks_per_node )) - # With ESMF threading, the model wants to use the full node - export APRUN_UFS="${launcher} -n ${ufs_ntasks}" - unset nnodes ufs_ntasks - -elif [[ "${step}" = "upp" ]]; then - - export NTHREADS_UPP=${NTHREADS1} - export OMP_NUM_THREADS="${NTHREADS_UPP}" - export APRUN_UPP="${APRUN_default}" - -elif [[ "${step}" = "atmos_products" ]]; then - - export USE_CFP="YES" # Use MPMD for downstream product generation - -elif [[ "${step}" = "oceanice_products" ]]; then - - export NTHREADS_OCNICEPOST=${NTHREADS1} - export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}" - -elif [[ "${step}" = "ecen" ]]; then - - export NTHREADS_ECEN=${NTHREADSmax} - export APRUN_ECEN="${APRUN_default}" - - export NTHREADS_CHGRES=${threads_per_task_chgres:-12} - if [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]]; then - export NTHREADS_CHGRES=${max_tasks_per_node} - fi - export APRUN_CHGRES="time" - - export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} - if [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]]; then - export NTHREADS_CALCINC=${max_threads_per_task} - fi - export APRUN_CALCINC="${APRUN_default}" - -elif [[ "${step}" = "esfc" ]]; then - - export NTHREADS_ESFC=${NTHREADSmax} - export APRUN_ESFC="${APRUN_default}" - - export NTHREADS_CYCLE=${threads_per_task_cycle:-14} - if [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]]; then - export NTHREADS_CYCLE=${max_tasks_per_node} - fi - export APRUN_CYCLE="${APRUN_default}" - -elif [[ "${step}" = "epos" ]]; then - - export NTHREADS_EPOS=${NTHREADSmax} - export APRUN_EPOS="${APRUN_default}" - -elif [[ "${step}" = "fit2obs" ]]; then - - export NTHREADS_FIT2OBS=${NTHREADS1} - export MPIRUN="${APRUN_default}" - -fi diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua deleted file mode 100644 index 56735a20575..00000000000 --- a/modulefiles/module_base.jet.lua +++ /dev/null @@ -1,58 +0,0 @@ -help([[ -Load environment to run GFS on Jet -]]) - -local spack_mod_path=(os.getenv("spack_mod_path") or "None") -prepend_path("MODULEPATH", spack_mod_path) - -load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) -load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None"))) -load(pathJoin("python", (os.getenv("python_ver") or "None"))) - -load(pathJoin("hpss", (os.getenv("hpss_ver") or "None"))) -load(pathJoin("gempak", (os.getenv("gempak_ver") or "None"))) -load(pathJoin("ncl", (os.getenv("ncl_ver") or "None"))) -load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) -load(pathJoin("libpng", (os.getenv("libpng_ver") or "None"))) -load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) -load(pathJoin("R", (os.getenv("R_ver") or "None"))) - -load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) -load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) -load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) - -load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) -load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) -load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None"))) -load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None"))) -load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None"))) -load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) -load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) -load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) -load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) -load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) -load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) -load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) -load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) -load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None"))) -load(pathJoin("met", (os.getenv("met_ver") or "None"))) -load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) -load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) - --- Adding perl as a module; With Rocky8, perl packages will not be from the OS -load(pathJoin("perl", (os.getenv("perl_ver") or "None"))) - -setenv("WGRIB2","wgrib2") - --- Stop gap fix for wgrib with spack-stack 1.6.0 --- TODO Remove this when spack-stack issue #1097 is resolved -setenv("WGRIB","wgrib") -setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) - -prepend_path("MODULEPATH", pathJoin("/lfs5/HFIP/hfv3gfs/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) -load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) - -prepend_path("MODULEPATH", pathJoin("/lfs5/HFIP/hfv3gfs/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) -load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) - -whatis("Description: GFS run environment") diff --git a/modulefiles/module_base.s4.lua b/modulefiles/module_base.s4.lua deleted file mode 100644 index 835249fb853..00000000000 --- a/modulefiles/module_base.s4.lua +++ /dev/null @@ -1,53 +0,0 @@ -help([[ -Load environment to run GFS on S4 -]]) - -local spack_mod_path=(os.getenv("spack_mod_path") or "None") -prepend_path("MODULEPATH", spack_mod_path) - -load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) -load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None"))) -load(pathJoin("python", (os.getenv("python_ver") or "None"))) - -load(pathJoin("ncl", (os.getenv("ncl_ver") or "None"))) -load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) -load(pathJoin("libpng", (os.getenv("libpng_ver") or "None"))) -load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) - -load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) -load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) -load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) - -load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) -load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) -load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None"))) -load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None"))) -load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None"))) -load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) -load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) -load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) -load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) -load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) -load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) -load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) -load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) -load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None"))) -load(pathJoin("met", (os.getenv("met_ver") or "None"))) -load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) -load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) - -setenv("WGRIB2","wgrib2") - --- Stop gap fix for wgrib with spack-stack 1.6.0 --- TODO Remove this when spack-stack issue #1097 is resolved -setenv("WGRIB","wgrib") -setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) - ---prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) -prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) -load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) - -prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) -load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) - -whatis("Description: GFS run environment") diff --git a/modulefiles/module_gwsetup.jet.lua b/modulefiles/module_gwsetup.jet.lua deleted file mode 100644 index a722c812a56..00000000000 --- a/modulefiles/module_gwsetup.jet.lua +++ /dev/null @@ -1,18 +0,0 @@ -help([[ -Load environment to run GFS workflow setup scripts on Jet -]]) - -load(pathJoin("rocoto")) - -prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/gsi-addon-intel/install/modulefiles/Core") - -local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" -local python_ver=os.getenv("python_ver") or "3.11.6" - -load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("python", python_ver)) -load("py-jinja2") -load("py-pyyaml") -load("py-numpy") - -whatis("Description: GFS run setup environment") diff --git a/modulefiles/module_gwsetup.s4.lua b/modulefiles/module_gwsetup.s4.lua deleted file mode 100644 index 77a647006ff..00000000000 --- a/modulefiles/module_gwsetup.s4.lua +++ /dev/null @@ -1,19 +0,0 @@ -help([[ -Load environment to run GFS workflow setup scripts on S4 -]]) - -load(pathJoin("rocoto","1.3.5")) -load(pathJoin("git","2.30.0")) - -prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") - -local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" -local python_ver=os.getenv("python_ver") or "3.11.6" - -load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("python", python_ver)) -load("py-jinja2") -load("py-pyyaml") -load("py-numpy") - -whatis("Description: GFS run setup environment") diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 88268ba9364..61c8555840d 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -20,27 +20,8 @@ case ${machine} in "HERA") max_tasks_per_node=40;; "ORION") max_tasks_per_node=40;; "HERCULES") max_tasks_per_node=80;; - "JET") - case ${PARTITION_BATCH} in - "xjet") max_tasks_per_node=24;; - "vjet" | "sjet") max_tasks_per_node=16;; - "kjet") max_tasks_per_node=40;; - *) - echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}" - exit 3 - esac - ;; "GAEAC5") max_tasks_per_node=128;; "GAEAC6") max_tasks_per_node=192;; - "S4") - case ${PARTITION_BATCH} in - "s4") max_tasks_per_node=32;; - "ivy") max_tasks_per_node=20;; - *) - echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}" - exit 3 - esac - ;; "AWSPW") export PARTITION_BATCH="compute" max_tasks_per_node=48 diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 27d64626b00..3302bfc8cd9 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -67,49 +67,6 @@ case ${machine} in # shellcheck disable=SC2034 mem_node_max="500GB" ;; - "JET") - case ${PARTITION_BATCH} in - "xjet") - max_tasks_per_node=24 - # shellcheck disable=SC2034 - mem_node_max="61GB" - ;; - "vjet") - max_tasks_per_node=16 - # shellcheck disable=SC2034 - mem_node_max="61GB" - ;; - "sjet") - max_tasks_per_node=16 - # shellcheck disable=SC2034 - mem_node_max="29GB" - ;; - "kjet") - max_tasks_per_node=40 - # shellcheck disable=SC2034 - mem_node_max="88GB" - ;; - *) - echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}" - exit 3 - esac - ;; - "S4") - case ${PARTITION_BATCH} in - "s4") max_tasks_per_node=32 - # shellcheck disable=SC2034 - mem_node_max="168GB" - ;; - "ivy") - max_tasks_per_node=20 - # shellcheck disable=SC2034 - mem_node_max="128GB" - ;; - *) - echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}" - exit 3 - esac - ;; "AWSPW") export PARTITION_BATCH="compute" npe_node_max=48 diff --git a/parm/config/gfs/config.resources.JET b/parm/config/gfs/config.resources.JET deleted file mode 100644 index 536da8a4f22..00000000000 --- a/parm/config/gfs/config.resources.JET +++ /dev/null @@ -1,57 +0,0 @@ -#! /usr/bin/env bash - -# Jet-specific job resources - -case ${step} in - "prep") - # Run on 7 nodes for memory requirement - tasks_per_node=2 - ;; - - "anal") - if [[ "${CASE}" == "C384" ]]; then - export ntasks=270 - export threads_per_task=8 - export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) - fi - ;; - - "eobs") - if [[ "${PARTITION_BATCH}" == "xjet" ]]; then - # The number of tasks and cores used must be the same for eobs - # See https://github.com/NOAA-EMC/global-workflow/issues/2092 for details - # This would also be an issues for vjet and sjet if anyone runs on those nodes. - export tasks_per_node=10 - fi - ;; - - "eupd") - case "${CASE}" in - "C384") - export ntasks=80 - ;; - "C192" | "C96" | "C48") - export threads_per_task=4 - ;; - *) - ;; - esac - export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) - ;; - - "ecen") - if [[ "${CASE}" == "C768" || "${CASE}" == "C1152" ]]; then export threads_per_task=6; fi - export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) - ;; - - "upp") - export memory="${mem_node_max}" - ;; - - "esfc") - export memory="${mem_node_max}" - ;; - - *) - ;; -esac diff --git a/parm/config/gfs/config.resources.S4 b/parm/config/gfs/config.resources.S4 deleted file mode 100644 index 86102e2a1fa..00000000000 --- a/parm/config/gfs/config.resources.S4 +++ /dev/null @@ -1,61 +0,0 @@ -#! /usr/bin/env bash - -# S4-specific job resources - -case ${step} in - "prep") - # Run on two nodes for memory requirement - # This may not be enough memory. Decrease tasks/node to 2 if necessary. - tasks_per_node=7 - ;; - - "anal") - case "${CASE}" in - "C384") - #Some of the intermediate data can be lost if the number of tasks - #per node does not match the number of reserved cores/node. - #On the S4-s4 partition, this is accomplished by increasing the task - #count to a multiple of 32 - if [[ ${PARTITION_BATCH} = "s4" ]]; then - export ntasks=416 - fi - #S4 is small, so run this task with just 1 thread - export threads_per_task=1 - export walltime="02:00:00" - ;; - "C192" | "C96" | "C48") - export threads_per_task=4 - if [[ ${PARTITION_BATCH} == "s4" ]]; then - export ntasks=88 - elif [[ ${PARTITION_BATCH} == "ivy" ]]; then - export ntasks=90 - fi - ;; - *) - ;; - esac - export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) - ;; - - "eobs") - # The number of tasks and cores used must be the same for eobs - # See https://github.com/NOAA-EMC/global-workflow/issues/2092 for details - # For S4, this is accomplished by running 10 tasks/node - export tasks_per_node=10 - ;; - - "eupd") - if [[ "${CASE}" == "C384" ]]; then - export ntasks=160 - export threads_per_task=2 - fi - export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) - ;; - - "ediag") - export memory="${mem_node_max}" - ;; - - *) - ;; -esac diff --git a/parm/config/sfs/config.resources b/parm/config/sfs/config.resources index f929c3eaefd..077b3ac8107 100644 --- a/parm/config/sfs/config.resources +++ b/parm/config/sfs/config.resources @@ -20,25 +20,6 @@ case ${machine} in "HERA") max_tasks_per_node=40;; "ORION") max_tasks_per_node=40;; "HERCULES") max_tasks_per_node=80;; - "JET") - case ${PARTITION_BATCH} in - "xjet") max_tasks_per_node=24;; - "vjet" | "sjet") max_tasks_per_node=16;; - "kjet") max_tasks_per_node=40;; - *) - echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}" - exit 3 - esac - ;; - "S4") - case ${PARTITION_BATCH} in - "s4") max_tasks_per_node=32;; - "ivy") max_tasks_per_node=20;; - *) - echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}" - exit 3 - esac - ;; "AWSPW") export PARTITION_BATCH="compute" max_tasks_per_node=48 diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 12391c398f0..56a163dc953 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -74,8 +74,6 @@ case "${machine}" in "hera") FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix" ;; "orion") FIX_DIR="/work2/noaa/global/role-global/fix" ;; "hercules") FIX_DIR="/work2/noaa/global/role-global/fix" ;; -"jet") FIX_DIR="/lfs5/HFIP/hfv3gfs/glopara/FIX/fix" ;; -"s4") FIX_DIR="/data/prod/glopara/fix" ;; "gaeac5") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; "gaeac6") FIX_DIR="/gpfs/f6/drsa-precip3/world-shared/role.glopara/fix" ;; "noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;; diff --git a/ush/calcanl_gfs.py b/ush/calcanl_gfs.py index f5e65f38245..3eeb077c3ed 100755 --- a/ush/calcanl_gfs.py +++ b/ush/calcanl_gfs.py @@ -242,13 +242,6 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, with open(CalcAnlDir + '/hosts', 'w') as hostfile: hostfile.write(hosts[ihost] + '\n') if launcher == 'srun': # need to write host per task not per node for slurm - # For xjet, each instance of chgres_inc must run on two nodes each - if os.getenv('SLURM_JOB_PARTITION', '') == 'xjet': - for a in range(0, 4): - hostfile.write(hosts[ihost] + '\n') - ihost += 1 - for a in range(0, 5): - hostfile.write(hosts[ihost] + '\n') for a in range(0, 12): # need 12 more of the same host for the 13 tasks for chgres_inc hostfile.write(hosts[ihost] + '\n') if launcher == 'srun': diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index e8807d1cdf8..041230c7397 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -26,7 +26,7 @@ source "${HOMEgfs}/versions/run.ver" module use "${HOMEgfs}/modulefiles" case "${MACHINE_ID}" in - "wcoss2" | "hera" | "orion" | "hercules" | "gaeac5" | "gaeac6" | "jet" | "s4" | "noaacloud") + "wcoss2" | "hera" | "orion" | "hercules" | "gaeac5" | "gaeac6" | "noaacloud") module load "module_base.${MACHINE_ID}" ;; *) diff --git a/ush/load_ufsda_modules.sh b/ush/load_ufsda_modules.sh index df44765f0bc..75b31d4d248 100755 --- a/ush/load_ufsda_modules.sh +++ b/ush/load_ufsda_modules.sh @@ -54,7 +54,7 @@ case "${MACHINE_ID}" in NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) export NETCDF ;; - ("jet" | "s4" | "acorn") + ("acorn") echo WARNING: UFSDA NOT SUPPORTED ON THIS PLATFORM ;; *) diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 0b2ffae4c9c..61948fa3995 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -3,15 +3,7 @@ set -u source "${HOMEgfs}/ush/detect_machine.sh" -if [[ ${MACHINE_ID} = jet* ]] ; then - # We are on NOAA Jet - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/lmod/init/bash - fi - export LMOD_SYSTEM_DEFAULT_MODULES=contrib - module reset - -elif [[ ${MACHINE_ID} = hera* ]] ; then +if [[ ${MACHINE_ID} = hera* ]] ; then # We are on NOAA Hera if ( ! eval module help > /dev/null 2>&1 ) ; then source /apps/lmod/lmod/init/bash @@ -41,13 +33,6 @@ elif [[ ${MACHINE_ID} = orion* ]] ; then module purge # reset causes issues on Orion sometimes. #set -u -elif [[ ${MACHINE_ID} = s4* ]] ; then - # We are on SSEC Wisconsin S4 - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /usr/share/lmod/lmod/init/bash - fi - export LMOD_SYSTEM_DEFAULT_MODULES=license_intel - module reset elif [[ ${MACHINE_ID} = wcoss2 ]]; then # We are on WCOSS2 diff --git a/versions/build.jet.ver b/versions/build.jet.ver deleted file mode 100644 index 319a8fa0a64..00000000000 --- a/versions/build.jet.ver +++ /dev/null @@ -1,5 +0,0 @@ -export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-intel -source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/build.s4.ver b/versions/build.s4.ver deleted file mode 100644 index e2641c124f0..00000000000 --- a/versions/build.s4.ver +++ /dev/null @@ -1,5 +0,0 @@ -export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.0 -export spack_env=gsi-addon-env -source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/data/prod/jedi/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.jet.ver b/versions/run.jet.ver deleted file mode 100644 index 90553fb2773..00000000000 --- a/versions/run.jet.ver +++ /dev/null @@ -1,18 +0,0 @@ -export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-intel - -export hpss_ver= -export ncl_ver=6.6.2 -export R_ver=4.0.2 -export gempak_ver=7.4.2 - -# Adding perl as a module; With Rocky8, perl packages will not be from the OS -export perl_ver=5.38.0 - -source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" - -# Local version of TC_tracker -export ens_tracker_ver=v1.1.15.7 - diff --git a/versions/run.s4.ver b/versions/run.s4.ver deleted file mode 100644 index 43e690e19b3..00000000000 --- a/versions/run.s4.ver +++ /dev/null @@ -1,8 +0,0 @@ -export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.0 -export spack_env=gsi-addon-env - -export ncl_ver=6.4.0-precompiled - -source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/data/prod/jedi/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"