From ba14b40aed264f648fcb0df7a6ac56ae6b6f352d Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Thu, 7 Apr 2022 08:31:52 -0400 Subject: [PATCH 1/7] Updates to port regional workflow to gaea --- modulefiles/tasks/gaea/make_grid.local | 6 +++ modulefiles/tasks/gaea/make_ics.local | 6 +++ modulefiles/tasks/gaea/make_lbcs.local | 6 +++ modulefiles/tasks/gaea/run_fcst.local | 6 +++ modulefiles/tasks/gaea/run_vx.local | 7 +++ ush/config_defaults.sh | 1 + ush/generate_FV3LAM_wflow.sh | 1 + ush/machine/gaea.sh | 73 ++++++++++++++++++++++++++ ush/templates/FV3LAM_wflow.xml | 42 +++++++++++++++ ush/valid_param_vals.sh | 2 +- 10 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 modulefiles/tasks/gaea/make_grid.local create mode 100644 modulefiles/tasks/gaea/make_ics.local create mode 100644 modulefiles/tasks/gaea/make_lbcs.local create mode 100644 modulefiles/tasks/gaea/run_fcst.local create mode 100644 modulefiles/tasks/gaea/run_vx.local create mode 100755 ush/machine/gaea.sh diff --git a/modulefiles/tasks/gaea/make_grid.local b/modulefiles/tasks/gaea/make_grid.local new file mode 100644 index 000000000..fbd52ffa0 --- /dev/null +++ b/modulefiles/tasks/gaea/make_grid.local @@ -0,0 +1,6 @@ +#%Module +module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles +module load rocoto +module load miniconda3 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/gaea/make_ics.local b/modulefiles/tasks/gaea/make_ics.local new file mode 100644 index 000000000..fbd52ffa0 --- /dev/null +++ b/modulefiles/tasks/gaea/make_ics.local @@ -0,0 +1,6 @@ +#%Module +module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles +module load rocoto +module load miniconda3 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/gaea/make_lbcs.local b/modulefiles/tasks/gaea/make_lbcs.local new file mode 100644 index 000000000..fbd52ffa0 --- /dev/null +++ b/modulefiles/tasks/gaea/make_lbcs.local @@ -0,0 +1,6 @@ +#%Module +module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles +module load rocoto +module load miniconda3 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/gaea/run_fcst.local b/modulefiles/tasks/gaea/run_fcst.local new file mode 100644 index 000000000..fbd52ffa0 --- /dev/null +++ b/modulefiles/tasks/gaea/run_fcst.local @@ -0,0 +1,6 @@ +#%Module +module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles +module load rocoto +module load miniconda3 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/gaea/run_vx.local b/modulefiles/tasks/gaea/run_vx.local new file mode 100644 index 000000000..929c2011f --- /dev/null +++ b/modulefiles/tasks/gaea/run_vx.local @@ -0,0 +1,7 @@ +#%Module + +module use -a /contrib/anaconda/modulefiles +module load intel/18.0.5.274 +module load anaconda/latest +module use -a /contrib/met/modulefiles/ +module load met/10.0.0 diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index be96316d7..11942c1d5 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1236,6 +1236,7 @@ DEBUG="FALSE" # #----------------------------------------------------------------------- # +EXTRA_SLURM_CMD="" MAKE_GRID_TN="make_grid" MAKE_OROG_TN="make_orog" MAKE_SFC_CLIMO_TN="make_sfc_climo" diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 83282b717..d15bf3d28 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -193,6 +193,7 @@ file (template_xml_fp): 'partition_fcst': ${PARTITION_FCST} 'queue_fcst': ${QUEUE_FCST} 'machine': ${MACHINE} + 'extra_slurm_cmd': ${EXTRA_SLURM_CMD} # # Workflow task names. # diff --git a/ush/machine/gaea.sh b/ush/machine/gaea.sh new file mode 100755 index 000000000..bdcddcc80 --- /dev/null +++ b/ush/machine/gaea.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +set -x + +function file_location() { + + # Return the default location of external model files on disk + + local external_file_fmt external_model location + + external_model=${1} + external_file_fmt=${2} + + case ${external_model} in + + "FV3GFS") + location='/lustre/f2/dev/Mark.Potts/EPIC/SRW/model_data/FV3GFS/${yyyymmdd}${hh}' + ;; + *) + print_info_msg"\ + External model \'${external_model}\' does not have a default + location on Hera. Will try to pull from HPSS" + ;; + + esac + echo ${location:-} +} + + +EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ + ${EXTRN_MDL_NAME_ICS} \ + ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ + ${EXTRN_MDL_NAME_LBCS} \ + ${FV3GFS_FILE_FMT_ICS})} + +# System scripts to source to initialize various commands within workflow +# scripts (e.g. "module"). +if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then + ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) +fi + + +# Commands to run at the start of each workflow task. +PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' + +# Architecture information +WORKFLOW_MANAGER="rocoto" +NCORES_PER_NODE=${NCORES_PER_NODE:-36} +SCHED=${SCHED:-"slurm"} +QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} +QUEUE_HPSS=${QUEUE_DEFAULT:-"normal"} +QUEUE_FCST=${QUEUE_DEFAULT:-"normal"} + +# UFS SRW App specific paths +FIXgsm=${FIXgsm:-"//lustre/f2/dev/Mark.Potts/EPIC/fix/fix_am"} +FIXaer=${FIXaer:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_aer"} +FIXlut=${FIXlut:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_sfc_climo"} + +RUN_CMD_SERIAL="time" +#Run Commands currently differ for GNU/openmpi +#RUN_CMD_UTILS='mpirun --mca btl tcp,vader,self -np $nprocs' +#RUN_CMD_FCST='mpirun --mca btl tcp,vader,self -np ${PE_MEMBER01}' +#RUN_CMD_POST='mpirun --mca btl tcp,vader,self -np $nprocs' +RUN_CMD_UTILS='srun --mpi=pmi2 -n $nprocs' +RUN_CMD_FCST='srun --mpi=pmi2 -n ${PE_MEMBER01}' +RUN_CMD_POST='srun --mpi=pmi2 -n $nprocs' + +# MET Installation Locations +# MET Plus is not yet supported on noaacloud + diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 48e53d81a..d91070645 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -22,6 +22,7 @@ Parameters needed by the job scheduler. + @@ -184,6 +185,7 @@ MODULES_RUN_TASK_FP script. {%- endif %} {{ wtime_make_grid }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &MAKE_GRID_TN; &LOGDIR;/&MAKE_GRID_TN;.log @@ -207,6 +209,7 @@ MODULES_RUN_TASK_FP script. {%- endif %} {{ wtime_make_orog }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &MAKE_OROG_TN; &LOGDIR;/&MAKE_OROG_TN;.log @@ -234,6 +237,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_make_sfc_climo }}:ppn={{ ppn_make_sfc_climo }} {{ wtime_make_sfc_climo }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &MAKE_SFC_CLIMO_TN; &LOGDIR;/&MAKE_SFC_CLIMO_TN;.log @@ -271,6 +275,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_get_extrn_ics }}:ppn={{ ppn_get_extrn_ics }} {{ wtime_get_extrn_ics }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &GET_EXTRN_ICS_TN; &LOGDIR;/&GET_EXTRN_ICS_TN;_@Y@m@d@H.log @@ -298,6 +303,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_get_extrn_lbcs }}:ppn={{ ppn_get_extrn_lbcs }} {{ wtime_get_extrn_lbcs }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &GET_EXTRN_LBCS_TN; &LOGDIR;/&GET_EXTRN_LBCS_TN;_@Y@m@d@H.log @@ -332,6 +338,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_make_ics }}:ppn={{ ppn_make_ics }} {{ wtime_make_ics }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &MAKE_ICS_TN;{{ uscore_ensmem_name }} &LOGDIR;/&MAKE_ICS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -376,6 +383,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_make_lbcs }}:ppn={{ ppn_make_lbcs }} {{ wtime_make_lbcs }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &MAKE_LBCS_TN;{{ uscore_ensmem_name }} &LOGDIR;/&MAKE_LBCS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -423,6 +431,7 @@ MODULES_RUN_TASK_FP script. {%- else %} {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; {%- endif %} {{ wtime_run_fcst }} &RUN_FCST_TN;{{ uscore_ensmem_name }} @@ -475,6 +484,7 @@ later below for other output times. {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -529,6 +539,7 @@ for other output times. {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -588,6 +599,7 @@ always zero). {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; {%- if sub_hourly_post %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -656,6 +668,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -699,6 +712,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_get_obs_ccpa }}:ppn={{ ppn_get_obs_ccpa }} {{ wtime_get_obs_ccpa }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &GET_OBS_CCPA_TN; &LOGDIR;/&GET_OBS_CCPA_TN;_@Y@m@d@H.log @@ -729,6 +743,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_get_obs_mrms }}:ppn={{ ppn_get_obs_mrms }} {{ wtime_get_obs_mrms }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &GET_OBS_MRMS_TN; &LOGDIR;/&GET_OBS_MRMS_TN;_@Y@m@d@H.log @@ -760,6 +775,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_get_obs_ndas }}:ppn={{ ppn_get_obs_ndas }} {{ wtime_get_obs_ndas }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &GET_OBS_NDAS_TN; &LOGDIR;/&GET_OBS_NDAS_TN;_@Y@m@d@H.log @@ -786,6 +802,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_GRIDSTAT_TN; &LOGDIR;/&VX_GRIDSTAT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -836,6 +853,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_GRIDSTAT_REFC_TN; &LOGDIR;/&VX_GRIDSTAT_REFC_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -885,6 +903,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_GRIDSTAT_RETOP_TN; &LOGDIR;/&VX_GRIDSTAT_RETOP_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -934,6 +953,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_GRIDSTAT_03h_TN; &LOGDIR;/&VX_GRIDSTAT_03h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -969,6 +989,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_GRIDSTAT_06h_TN; &LOGDIR;/&VX_GRIDSTAT_06h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1004,6 +1025,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_GRIDSTAT_24h_TN; &LOGDIR;/&VX_GRIDSTAT_24h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1039,6 +1061,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_pointstat }}:ppn={{ ppn_vx_pointstat }} {{ wtime_vx_pointstat }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_POINTSTAT_TN; &LOGDIR;/&VX_POINTSTAT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1091,6 +1114,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_TN; &LOGDIR;/&VX_ENSGRID_TN;_@Y@m@d@H.log @@ -1123,6 +1147,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_03h_TN; &LOGDIR;/&VX_ENSGRID_03h_TN;_@Y@m@d@H.log @@ -1155,6 +1180,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_06h_TN; &LOGDIR;/&VX_ENSGRID_06h_TN;_@Y@m@d@H.log @@ -1187,6 +1213,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_24h_TN; &LOGDIR;/&VX_ENSGRID_24h_TN;_@Y@m@d@H.log @@ -1218,6 +1245,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_REFC_TN; &LOGDIR;/&VX_ENSGRID_REFC_TN;_@Y@m@d@H.log @@ -1248,6 +1276,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_RETOP_TN; &LOGDIR;/&VX_ENSGRID_RETOP_TN;_@Y@m@d@H.log @@ -1277,6 +1306,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_MEAN_TN; &LOGDIR;/&VX_ENSGRID_MEAN_TN;_@Y@m@d@H.log @@ -1308,6 +1338,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_PROB_TN; &LOGDIR;/&VX_ENSGRID_PROB_TN;_@Y@m@d@H.log @@ -1339,6 +1370,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_MEAN_03h_TN; &LOGDIR;/&VX_ENSGRID_MEAN_03h_TN;_@Y@m@d@H.log @@ -1370,6 +1402,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_PROB_03h_TN; &LOGDIR;/&VX_ENSGRID_PROB_03h_TN;_@Y@m@d@H.log @@ -1402,6 +1435,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_MEAN_06h_TN; &LOGDIR;/&VX_ENSGRID_MEAN_06h_TN;_@Y@m@d@H.log @@ -1433,6 +1467,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_PROB_06h_TN; &LOGDIR;/&VX_ENSGRID_PROB_06h_TN;_@Y@m@d@H.log @@ -1465,6 +1500,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_MEAN_24h_TN; &LOGDIR;/&VX_ENSGRID_MEAN_24h_TN;_@Y@m@d@H.log @@ -1496,6 +1532,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_PROB_24h_TN; &LOGDIR;/&VX_ENSGRID_PROB_24h_TN;_@Y@m@d@H.log @@ -1527,6 +1564,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_PROB_REFC_TN; &LOGDIR;/&VX_ENSGRID_PROB_REFC_TN;_@Y@m@d@H.log @@ -1557,6 +1595,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSGRID_PROB_RETOP_TN; &LOGDIR;/&VX_ENSGRID_PROB_RETOP_TN;_@Y@m@d@H.log @@ -1588,6 +1627,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_enspoint }}:ppn={{ ppn_vx_enspoint }} {{ wtime_vx_enspoint }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSPOINT_TN; &LOGDIR;/&VX_ENSPOINT_TN;_@Y@m@d@H.log @@ -1616,6 +1656,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_enspoint_mean }}:ppn={{ ppn_vx_enspoint_mean }} {{ wtime_vx_enspoint_mean }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSPOINT_MEAN_TN; &LOGDIR;/&VX_ENSPOINT_MEAN_TN;_@Y@m@d@H.log @@ -1644,6 +1685,7 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_enspoint_prob }}:ppn={{ ppn_vx_enspoint_prob }} {{ wtime_vx_enspoint_prob }} &NCORES_PER_NODE; + &EXTRA_SLURM_CMD; &VX_ENSPOINT_PROB_TN; &LOGDIR;/&VX_ENSPOINT_PROB_TN;_@Y@m@d@H.log diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 1ea3a86b8..814f0fb82 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -4,7 +4,7 @@ valid_vals_RUN_ENVIR=("nco" "community") valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DEBUG=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_MACHINE=("WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS" "NOAACLOUD" "SINGULARITY") +valid_vals_MACHINE=("WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS" "NOAACLOUD" "SINGULARITY" "GAEA") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_FCST_MODEL=("ufs-weather-model" "fv3gfs_aqm") valid_vals_WORKFLOW_MANAGER=("rocoto" "none") From e405395d95a9dc046d3c23a7d4e7ab4495c1a9f8 Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Thu, 7 Apr 2022 15:52:15 +0000 Subject: [PATCH 2/7] Temp change with -v as batch option --- ush/config_defaults.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 11942c1d5..29d88027b 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1236,7 +1236,7 @@ DEBUG="FALSE" # #----------------------------------------------------------------------- # -EXTRA_SLURM_CMD="" +EXTRA_SLURM_CMD="-v" MAKE_GRID_TN="make_grid" MAKE_OROG_TN="make_orog" MAKE_SFC_CLIMO_TN="make_sfc_climo" From 27dc7df05965c7ebd97e4a2c7cd52daae2eae5b3 Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Thu, 7 Apr 2022 18:03:06 +0000 Subject: [PATCH 3/7] new fixes for gaea/slurm --- ush/config_defaults.sh | 2 +- ush/templates/FV3LAM_wflow.xml | 82 ++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 29d88027b..11942c1d5 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1236,7 +1236,7 @@ DEBUG="FALSE" # #----------------------------------------------------------------------- # -EXTRA_SLURM_CMD="-v" +EXTRA_SLURM_CMD="" MAKE_GRID_TN="make_grid" MAKE_OROG_TN="make_orog" MAKE_SFC_CLIMO_TN="make_sfc_climo" diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index d91070645..8c401593a 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -185,7 +185,9 @@ MODULES_RUN_TASK_FP script. {%- endif %} {{ wtime_make_grid }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &MAKE_GRID_TN; &LOGDIR;/&MAKE_GRID_TN;.log @@ -209,7 +211,9 @@ MODULES_RUN_TASK_FP script. {%- endif %} {{ wtime_make_orog }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &MAKE_OROG_TN; &LOGDIR;/&MAKE_OROG_TN;.log @@ -237,7 +241,9 @@ MODULES_RUN_TASK_FP script. {{ nnodes_make_sfc_climo }}:ppn={{ ppn_make_sfc_climo }} {{ wtime_make_sfc_climo }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &MAKE_SFC_CLIMO_TN; &LOGDIR;/&MAKE_SFC_CLIMO_TN;.log @@ -275,7 +281,9 @@ MODULES_RUN_TASK_FP script. {{ nnodes_get_extrn_ics }}:ppn={{ ppn_get_extrn_ics }} {{ wtime_get_extrn_ics }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &GET_EXTRN_ICS_TN; &LOGDIR;/&GET_EXTRN_ICS_TN;_@Y@m@d@H.log @@ -303,7 +311,9 @@ MODULES_RUN_TASK_FP script. {{ nnodes_get_extrn_lbcs }}:ppn={{ ppn_get_extrn_lbcs }} {{ wtime_get_extrn_lbcs }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &GET_EXTRN_LBCS_TN; &LOGDIR;/&GET_EXTRN_LBCS_TN;_@Y@m@d@H.log @@ -338,7 +348,9 @@ MODULES_RUN_TASK_FP script. {{ nnodes_make_ics }}:ppn={{ ppn_make_ics }} {{ wtime_make_ics }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &MAKE_ICS_TN;{{ uscore_ensmem_name }} &LOGDIR;/&MAKE_ICS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -383,7 +395,9 @@ MODULES_RUN_TASK_FP script. {{ nnodes_make_lbcs }}:ppn={{ ppn_make_lbcs }} {{ wtime_make_lbcs }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &MAKE_LBCS_TN;{{ uscore_ensmem_name }} &LOGDIR;/&MAKE_LBCS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -431,6 +445,8 @@ MODULES_RUN_TASK_FP script. {%- else %} {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} &NCORES_PER_NODE; + {%- endif %} + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; {%- endif %} {{ wtime_run_fcst }} @@ -484,7 +500,9 @@ later below for other output times. {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -539,7 +557,9 @@ for other output times. {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -599,7 +619,9 @@ always zero). {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} {%- if sub_hourly_post %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -668,7 +690,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -712,7 +736,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_get_obs_ccpa }}:ppn={{ ppn_get_obs_ccpa }} {{ wtime_get_obs_ccpa }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &GET_OBS_CCPA_TN; &LOGDIR;/&GET_OBS_CCPA_TN;_@Y@m@d@H.log @@ -743,7 +769,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_get_obs_mrms }}:ppn={{ ppn_get_obs_mrms }} {{ wtime_get_obs_mrms }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &GET_OBS_MRMS_TN; &LOGDIR;/&GET_OBS_MRMS_TN;_@Y@m@d@H.log @@ -775,7 +803,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_get_obs_ndas }}:ppn={{ ppn_get_obs_ndas }} {{ wtime_get_obs_ndas }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &GET_OBS_NDAS_TN; &LOGDIR;/&GET_OBS_NDAS_TN;_@Y@m@d@H.log @@ -802,7 +832,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_GRIDSTAT_TN; &LOGDIR;/&VX_GRIDSTAT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -853,7 +885,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_GRIDSTAT_REFC_TN; &LOGDIR;/&VX_GRIDSTAT_REFC_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -903,7 +937,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_GRIDSTAT_RETOP_TN; &LOGDIR;/&VX_GRIDSTAT_RETOP_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -953,7 +989,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_GRIDSTAT_03h_TN; &LOGDIR;/&VX_GRIDSTAT_03h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -989,7 +1027,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_GRIDSTAT_06h_TN; &LOGDIR;/&VX_GRIDSTAT_06h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1025,7 +1065,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_GRIDSTAT_24h_TN; &LOGDIR;/&VX_GRIDSTAT_24h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1061,7 +1103,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_pointstat }}:ppn={{ ppn_vx_pointstat }} {{ wtime_vx_pointstat }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_POINTSTAT_TN; &LOGDIR;/&VX_POINTSTAT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1114,7 +1158,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_TN; &LOGDIR;/&VX_ENSGRID_TN;_@Y@m@d@H.log @@ -1147,7 +1193,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_03h_TN; &LOGDIR;/&VX_ENSGRID_03h_TN;_@Y@m@d@H.log @@ -1180,7 +1228,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_06h_TN; &LOGDIR;/&VX_ENSGRID_06h_TN;_@Y@m@d@H.log @@ -1213,7 +1263,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_24h_TN; &LOGDIR;/&VX_ENSGRID_24h_TN;_@Y@m@d@H.log @@ -1245,7 +1297,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_REFC_TN; &LOGDIR;/&VX_ENSGRID_REFC_TN;_@Y@m@d@H.log @@ -1276,7 +1330,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_RETOP_TN; &LOGDIR;/&VX_ENSGRID_RETOP_TN;_@Y@m@d@H.log @@ -1306,7 +1362,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_MEAN_TN; &LOGDIR;/&VX_ENSGRID_MEAN_TN;_@Y@m@d@H.log @@ -1338,7 +1396,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_PROB_TN; &LOGDIR;/&VX_ENSGRID_PROB_TN;_@Y@m@d@H.log @@ -1370,7 +1430,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_MEAN_03h_TN; &LOGDIR;/&VX_ENSGRID_MEAN_03h_TN;_@Y@m@d@H.log @@ -1402,7 +1464,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_PROB_03h_TN; &LOGDIR;/&VX_ENSGRID_PROB_03h_TN;_@Y@m@d@H.log @@ -1435,7 +1499,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_MEAN_06h_TN; &LOGDIR;/&VX_ENSGRID_MEAN_06h_TN;_@Y@m@d@H.log @@ -1467,7 +1533,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_PROB_06h_TN; &LOGDIR;/&VX_ENSGRID_PROB_06h_TN;_@Y@m@d@H.log @@ -1500,7 +1568,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_MEAN_24h_TN; &LOGDIR;/&VX_ENSGRID_MEAN_24h_TN;_@Y@m@d@H.log @@ -1532,7 +1602,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_PROB_24h_TN; &LOGDIR;/&VX_ENSGRID_PROB_24h_TN;_@Y@m@d@H.log @@ -1564,7 +1636,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_PROB_REFC_TN; &LOGDIR;/&VX_ENSGRID_PROB_REFC_TN;_@Y@m@d@H.log @@ -1595,7 +1669,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSGRID_PROB_RETOP_TN; &LOGDIR;/&VX_ENSGRID_PROB_RETOP_TN;_@Y@m@d@H.log @@ -1627,7 +1703,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_enspoint }}:ppn={{ ppn_vx_enspoint }} {{ wtime_vx_enspoint }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSPOINT_TN; &LOGDIR;/&VX_ENSPOINT_TN;_@Y@m@d@H.log @@ -1656,7 +1734,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_enspoint_mean }}:ppn={{ ppn_vx_enspoint_mean }} {{ wtime_vx_enspoint_mean }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSPOINT_MEAN_TN; &LOGDIR;/&VX_ENSPOINT_MEAN_TN;_@Y@m@d@H.log @@ -1685,7 +1765,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_enspoint_prob }}:ppn={{ ppn_vx_enspoint_prob }} {{ wtime_vx_enspoint_prob }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} &EXTRA_SLURM_CMD; + {%- endif %} &VX_ENSPOINT_PROB_TN; &LOGDIR;/&VX_ENSPOINT_PROB_TN;_@Y@m@d@H.log From c691fc26579b9cc4fb68e89995b29929f190cd3a Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Fri, 8 Apr 2022 12:02:46 -0400 Subject: [PATCH 4/7] Updated time for make lbcs --- ush/machine/gaea.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ush/machine/gaea.sh b/ush/machine/gaea.sh index bdcddcc80..619768e55 100755 --- a/ush/machine/gaea.sh +++ b/ush/machine/gaea.sh @@ -46,11 +46,13 @@ PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' # Architecture information WORKFLOW_MANAGER="rocoto" -NCORES_PER_NODE=${NCORES_PER_NODE:-36} +EXTRA_SLURM_CMD="-M c3" +NCORES_PER_NODE=${NCORES_PER_NODE:-32} SCHED=${SCHED:-"slurm"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} QUEUE_HPSS=${QUEUE_DEFAULT:-"normal"} QUEUE_FCST=${QUEUE_DEFAULT:-"normal"} +WTIME_MAKE_LBCS="00:60:00" # UFS SRW App specific paths FIXgsm=${FIXgsm:-"//lustre/f2/dev/Mark.Potts/EPIC/fix/fix_am"} From bcc437f733b27e89ca02c49c16502f753fecbd00 Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Wed, 20 Apr 2022 13:48:18 -0400 Subject: [PATCH 5/7] added TEST data directory path --- ush/machine/gaea.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ush/machine/gaea.sh b/ush/machine/gaea.sh index 619768e55..cedb0da23 100755 --- a/ush/machine/gaea.sh +++ b/ush/machine/gaea.sh @@ -72,4 +72,10 @@ RUN_CMD_POST='srun --mpi=pmi2 -n $nprocs' # MET Installation Locations # MET Plus is not yet supported on noaacloud +# Test Data Locations +#TEST_PREGEN_BASEDIR="/scratch2/BMC/det/UFS_SRW_app/FV3LAM_pregen" +#TEST_COMINgfs="/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS" +TEST_EXTRN_MDL_SOURCE_BASEDIR="/lustre/f2/dev/Mark.Potts/EPIC/SRW/model_data" +#TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" +#TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" From 197bb1bc080441b09d906152fa0278d38499d089 Mon Sep 17 00:00:00 2001 From: Mark Potts <33099090+mark-a-potts@users.noreply.github.com> Date: Wed, 20 Apr 2022 13:54:52 -0400 Subject: [PATCH 6/7] Update gaea.sh --- ush/machine/gaea.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ush/machine/gaea.sh b/ush/machine/gaea.sh index cedb0da23..9ba999449 100755 --- a/ush/machine/gaea.sh +++ b/ush/machine/gaea.sh @@ -73,9 +73,5 @@ RUN_CMD_POST='srun --mpi=pmi2 -n $nprocs' # MET Installation Locations # MET Plus is not yet supported on noaacloud # Test Data Locations -#TEST_PREGEN_BASEDIR="/scratch2/BMC/det/UFS_SRW_app/FV3LAM_pregen" -#TEST_COMINgfs="/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS" TEST_EXTRN_MDL_SOURCE_BASEDIR="/lustre/f2/dev/Mark.Potts/EPIC/SRW/model_data" -#TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" -#TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" From 128d1f1cc444de75c6095370de11cf93c2ff4cb4 Mon Sep 17 00:00:00 2001 From: Mark Potts Date: Thu, 21 Apr 2022 15:00:09 -0400 Subject: [PATCH 7/7] fixes for PR --- ush/config_defaults.sh | 9 +++- ush/generate_FV3LAM_wflow.sh | 2 +- ush/machine/gaea.sh | 11 +---- ush/templates/FV3LAM_wflow.xml | 84 +++++++++++++++++----------------- 4 files changed, 53 insertions(+), 53 deletions(-) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 11942c1d5..3e2450798 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1236,7 +1236,6 @@ DEBUG="FALSE" # #----------------------------------------------------------------------- # -EXTRA_SLURM_CMD="" MAKE_GRID_TN="make_grid" MAKE_OROG_TN="make_orog" MAKE_SFC_CLIMO_TN="make_sfc_climo" @@ -1691,6 +1690,14 @@ MAXTRIES_VX_ENSGRID_PROB_RETOP="1" MAXTRIES_VX_ENSPOINT="1" MAXTRIES_VX_ENSPOINT_MEAN="1" MAXTRIES_VX_ENSPOINT_PROB="1" + +# +#----------------------------------------------------------------------- +# +# Allows an extra parameter to be passed to slurm via XML Native +# command +# +SLURM_NATIVE_CMD="" # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index d15bf3d28..8af7497b0 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -193,7 +193,7 @@ file (template_xml_fp): 'partition_fcst': ${PARTITION_FCST} 'queue_fcst': ${QUEUE_FCST} 'machine': ${MACHINE} - 'extra_slurm_cmd': ${EXTRA_SLURM_CMD} + 'slurm_native_cmd': ${SLURM_NATIVE_CMD} # # Workflow task names. # diff --git a/ush/machine/gaea.sh b/ush/machine/gaea.sh index 9ba999449..9bad0af74 100755 --- a/ush/machine/gaea.sh +++ b/ush/machine/gaea.sh @@ -1,7 +1,5 @@ #!/bin/bash -set -x - function file_location() { # Return the default location of external model files on disk @@ -16,11 +14,6 @@ function file_location() { "FV3GFS") location='/lustre/f2/dev/Mark.Potts/EPIC/SRW/model_data/FV3GFS/${yyyymmdd}${hh}' ;; - *) - print_info_msg"\ - External model \'${external_model}\' does not have a default - location on Hera. Will try to pull from HPSS" - ;; esac echo ${location:-} @@ -46,7 +39,7 @@ PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' # Architecture information WORKFLOW_MANAGER="rocoto" -EXTRA_SLURM_CMD="-M c3" +SLURM_NATIVE_CMD="-M c3" NCORES_PER_NODE=${NCORES_PER_NODE:-32} SCHED=${SCHED:-"slurm"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} @@ -71,7 +64,7 @@ RUN_CMD_FCST='srun --mpi=pmi2 -n ${PE_MEMBER01}' RUN_CMD_POST='srun --mpi=pmi2 -n $nprocs' # MET Installation Locations -# MET Plus is not yet supported on noaacloud +# MET Plus is not yet supported on gaea # Test Data Locations TEST_EXTRN_MDL_SOURCE_BASEDIR="/lustre/f2/dev/Mark.Potts/EPIC/SRW/model_data" diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 8c401593a..aecf38158 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -22,7 +22,7 @@ Parameters needed by the job scheduler. - + @@ -186,7 +186,7 @@ MODULES_RUN_TASK_FP script. {{ wtime_make_grid }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &MAKE_GRID_TN; &LOGDIR;/&MAKE_GRID_TN;.log @@ -212,7 +212,7 @@ MODULES_RUN_TASK_FP script. {{ wtime_make_orog }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &MAKE_OROG_TN; &LOGDIR;/&MAKE_OROG_TN;.log @@ -242,7 +242,7 @@ MODULES_RUN_TASK_FP script. {{ wtime_make_sfc_climo }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &MAKE_SFC_CLIMO_TN; &LOGDIR;/&MAKE_SFC_CLIMO_TN;.log @@ -282,7 +282,7 @@ MODULES_RUN_TASK_FP script. {{ wtime_get_extrn_ics }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &GET_EXTRN_ICS_TN; &LOGDIR;/&GET_EXTRN_ICS_TN;_@Y@m@d@H.log @@ -312,7 +312,7 @@ MODULES_RUN_TASK_FP script. {{ wtime_get_extrn_lbcs }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &GET_EXTRN_LBCS_TN; &LOGDIR;/&GET_EXTRN_LBCS_TN;_@Y@m@d@H.log @@ -349,7 +349,7 @@ MODULES_RUN_TASK_FP script. {{ wtime_make_ics }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &MAKE_ICS_TN;{{ uscore_ensmem_name }} &LOGDIR;/&MAKE_ICS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -396,7 +396,7 @@ MODULES_RUN_TASK_FP script. {{ wtime_make_lbcs }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &MAKE_LBCS_TN;{{ uscore_ensmem_name }} &LOGDIR;/&MAKE_LBCS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -447,7 +447,7 @@ MODULES_RUN_TASK_FP script. &NCORES_PER_NODE; {%- endif %} {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} {{ wtime_run_fcst }} &RUN_FCST_TN;{{ uscore_ensmem_name }} @@ -501,7 +501,7 @@ later below for other output times. {{ wtime_run_post }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -558,7 +558,7 @@ for other output times. {{ wtime_run_post }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -620,7 +620,7 @@ always zero). {{ wtime_run_post }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} {%- if sub_hourly_post %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# @@ -691,7 +691,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_run_post }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -737,7 +737,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_get_obs_ccpa }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &GET_OBS_CCPA_TN; &LOGDIR;/&GET_OBS_CCPA_TN;_@Y@m@d@H.log @@ -770,7 +770,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_get_obs_mrms }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &GET_OBS_MRMS_TN; &LOGDIR;/&GET_OBS_MRMS_TN;_@Y@m@d@H.log @@ -804,7 +804,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_get_obs_ndas }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &GET_OBS_NDAS_TN; &LOGDIR;/&GET_OBS_NDAS_TN;_@Y@m@d@H.log @@ -833,7 +833,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_gridstat }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_GRIDSTAT_TN; &LOGDIR;/&VX_GRIDSTAT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -886,7 +886,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_gridstat }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_GRIDSTAT_REFC_TN; &LOGDIR;/&VX_GRIDSTAT_REFC_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -938,7 +938,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_gridstat }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_GRIDSTAT_RETOP_TN; &LOGDIR;/&VX_GRIDSTAT_RETOP_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -990,7 +990,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_gridstat }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_GRIDSTAT_03h_TN; &LOGDIR;/&VX_GRIDSTAT_03h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1028,7 +1028,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_gridstat }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_GRIDSTAT_06h_TN; &LOGDIR;/&VX_GRIDSTAT_06h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1066,7 +1066,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_gridstat }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_GRIDSTAT_24h_TN; &LOGDIR;/&VX_GRIDSTAT_24h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1104,7 +1104,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_pointstat }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_POINTSTAT_TN; &LOGDIR;/&VX_POINTSTAT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1159,7 +1159,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_TN; &LOGDIR;/&VX_ENSGRID_TN;_@Y@m@d@H.log @@ -1194,7 +1194,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_03h_TN; &LOGDIR;/&VX_ENSGRID_03h_TN;_@Y@m@d@H.log @@ -1229,7 +1229,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_06h_TN; &LOGDIR;/&VX_ENSGRID_06h_TN;_@Y@m@d@H.log @@ -1264,7 +1264,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_24h_TN; &LOGDIR;/&VX_ENSGRID_24h_TN;_@Y@m@d@H.log @@ -1298,7 +1298,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_REFC_TN; &LOGDIR;/&VX_ENSGRID_REFC_TN;_@Y@m@d@H.log @@ -1331,7 +1331,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_RETOP_TN; &LOGDIR;/&VX_ENSGRID_RETOP_TN;_@Y@m@d@H.log @@ -1363,7 +1363,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_MEAN_TN; &LOGDIR;/&VX_ENSGRID_MEAN_TN;_@Y@m@d@H.log @@ -1397,7 +1397,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_PROB_TN; &LOGDIR;/&VX_ENSGRID_PROB_TN;_@Y@m@d@H.log @@ -1431,7 +1431,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_MEAN_03h_TN; &LOGDIR;/&VX_ENSGRID_MEAN_03h_TN;_@Y@m@d@H.log @@ -1465,7 +1465,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_PROB_03h_TN; &LOGDIR;/&VX_ENSGRID_PROB_03h_TN;_@Y@m@d@H.log @@ -1500,7 +1500,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_MEAN_06h_TN; &LOGDIR;/&VX_ENSGRID_MEAN_06h_TN;_@Y@m@d@H.log @@ -1534,7 +1534,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_PROB_06h_TN; &LOGDIR;/&VX_ENSGRID_PROB_06h_TN;_@Y@m@d@H.log @@ -1569,7 +1569,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_MEAN_24h_TN; &LOGDIR;/&VX_ENSGRID_MEAN_24h_TN;_@Y@m@d@H.log @@ -1603,7 +1603,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_PROB_24h_TN; &LOGDIR;/&VX_ENSGRID_PROB_24h_TN;_@Y@m@d@H.log @@ -1637,7 +1637,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_PROB_REFC_TN; &LOGDIR;/&VX_ENSGRID_PROB_REFC_TN;_@Y@m@d@H.log @@ -1670,7 +1670,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_PROB_RETOP_TN; &LOGDIR;/&VX_ENSGRID_PROB_RETOP_TN;_@Y@m@d@H.log @@ -1704,7 +1704,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_enspoint }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSPOINT_TN; &LOGDIR;/&VX_ENSPOINT_TN;_@Y@m@d@H.log @@ -1735,7 +1735,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_enspoint_mean }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSPOINT_MEAN_TN; &LOGDIR;/&VX_ENSPOINT_MEAN_TN;_@Y@m@d@H.log @@ -1766,7 +1766,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_enspoint_prob }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} - &EXTRA_SLURM_CMD; + &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSPOINT_PROB_TN; &LOGDIR;/&VX_ENSPOINT_PROB_TN;_@Y@m@d@H.log