diff --git a/env/HERA.env b/env/HERA.env index 11f4c082c63..e8efa3f8b3c 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -130,14 +130,18 @@ elif [ $step = "eupd" ]; then elif [ $step = "fcst" ]; then + #PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs + if [[ $CDUMP == "gfs" ]]; then + npe_fcst=$npe_fcst_gfs + npe_node_fcst=$npe_node_fcst_gfs + nth_fv3=$nth_fv3_gfs + fi + nth_max=$(($npe_node_max / $npe_node_fcst)) export NTHREADS_FV3=${nth_fv3:-$nth_max} [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max export cores_per_node=$npe_node_max - if [[ $CDUMP == "gfs" ]]; then - npe_fcst=$npe_fcst_gfs - fi export APRUN_FV3="$launcher -n $npe_fcst" export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} diff --git a/env/JET.env b/env/JET.env index aa47a465307..113c2d1b377 100755 --- a/env/JET.env +++ b/env/JET.env @@ -99,6 +99,13 @@ elif [ $step = "eupd" ]; then elif [ $step = "fcst" ]; then + #PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs + if [[ $CDUMP == "gfs" ]]; then + npe_fcst=$npe_fcst_gfs + npe_node_fcst=$npe_node_fcst_gfs + nth_fv3=$nth_fv3_gfs + fi + nth_max=$(($npe_node_max / $npe_node_fcst)) export NTHREADS_FV3=${nth_fv3:-$nth_max} diff --git a/env/ORION.env b/env/ORION.env index 214493188ad..079f623f406 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -126,14 +126,18 @@ elif [ $step = "eupd" ]; then elif [ $step = "fcst" ]; then + #PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs + if [[ $CDUMP == "gfs" ]]; then + npe_fcst=$npe_fcst_gfs + npe_node_fcst=$npe_node_fcst_gfs + nth_fv3=$nth_fv3_gfs + fi + nth_max=$(($npe_node_max / $npe_node_fcst)) export NTHREADS_FV3=${nth_fv3:-$nth_max} [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max export cores_per_node=$npe_node_max - if [[ $CDUMP == "gfs" ]]; then - npe_fcst=$npe_fcst_gfs - fi export APRUN_FV3="$launcher -n $npe_fcst" export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} diff --git a/env/WCOSS_C.env b/env/WCOSS_C.env index f79586a8840..ea715a0ba67 100755 --- a/env/WCOSS_C.env +++ b/env/WCOSS_C.env @@ -115,17 +115,20 @@ elif [ $step = "eupd" ]; then elif [ $step = "fcst" ]; then + #PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs + if [[ $CDUMP == "gfs" ]]; then + npe_fcst=$npe_fcst_gfs + npe_node_fcst=$npe_node_fcst_gfs + nth_fv3=$nth_fv3_gfs + fi + nth_max=$(($npe_node_max / $npe_node_fcst)) export NTHREADS_FV3=${nth_fv3:-$nth_max} [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max export cores_per_node=$npe_node_max #export APRUN_FV3="$launcher -j 1 -n ${npe_fv3:-$npe_fcst} -N $npe_node_fcst -d $NTHREADS_FV3 -cc depth" - if [ $CDUMP = "gdas" ]; then - export APRUN_FV3="$launcher -j 1 -n ${npe_fcst} -N $npe_node_fcst -d $NTHREADS_FV3 -cc depth" - else - export APRUN_FV3="$launcher -j 1 -n ${npe_fcst_gfs} -N $npe_node_fcst -d $NTHREADS_FV3 -cc depth" - fi + export APRUN_FV3="$launcher -j 1 -n ${npe_fcst} -N $npe_node_fcst -d $NTHREADS_FV3 -cc depth" export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max diff --git a/env/WCOSS_DELL_P3.env b/env/WCOSS_DELL_P3.env index 785a44f69a9..b7761de2cc4 100755 --- a/env/WCOSS_DELL_P3.env +++ b/env/WCOSS_DELL_P3.env @@ -120,17 +120,20 @@ elif [ $step = "eupd" ]; then elif [ $step = "fcst" ]; then + #PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs + if [[ $CDUMP == "gfs" ]]; then + npe_fcst=$npe_fcst_gfs + npe_node_fcst=$npe_node_fcst_gfs + nth_fv3=$nth_fv3_gfs + fi + nth_max=$(($npe_node_max / $npe_node_fcst)) export NTHREADS_FV3=${nth_fv3:-$nth_max} [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max export cores_per_node=$npe_node_max - if [ $CDUMP = "gdas" ]; then - #export APRUN_FV3="$launcher ${npe_fv3:-${npe_fcst:-$PBS_NP}}" - export APRUN_FV3="$launcher ${npe_fcst:-$PBS_NP}" - else - export APRUN_FV3="$launcher ${npe_fcst_gfs:-$PBS_NP}" - fi + #export APRUN_FV3="$launcher ${npe_fv3:-${npe_fcst:-$PBS_NP}}" + export APRUN_FV3="$launcher ${npe_fcst:-$PBS_NP}" export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max export APRUN_REGRID_NEMSIO="$launcher $LEVS" diff --git a/parm/config/config.fcst b/parm/config/config.fcst index 1a246ba999a..ba3b2bb3c67 100755 --- a/parm/config/config.fcst +++ b/parm/config/config.fcst @@ -9,7 +9,6 @@ echo "BEGIN: config.fcst" # Source model specific information that is resolution dependent . $EXPDIR/config.fv3 $CASE -[[ "$CDUMP" == "gfs" ]] && export nth_fv3=$nth_fv3_gfs # Turn off waves if not used for this CDUMP case $WAVE_CDUMP in diff --git a/parm/config/config.resources b/parm/config/config.resources index d456e2783f9..b5c408506a2 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -209,8 +209,11 @@ elif [ $step = "fcst" ]; then NTASKS_TOT=$ATMPETS export nth_fcst=${nth_fv3:-2} + export nth_fcst_gfs=${nth_fv3_gfs:-2} export npe_node_fcst=$(echo "$npe_node_max / $nth_fcst" | bc) + export npe_node_fcst_gfs=$(echo "$npe_node_max / $nth_fcst_gfs" | bc) + if [[ "$machine" == "WCOSS_C" ]]; then export memory_fcst="1024M"; fi if [[ $DO_WAVE == "YES" ]]; then diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 786b759d4b0..5b9015613b2 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -122,7 +122,6 @@ FV3_GFS_predet(){ #file and the value of npe_node_fcst is not correctly defined when using more than #one thread and sets NTHREADS_FV3=1 even when the number of threads is appropraitely >1 #NTHREADS_FV3=${NTHREADS_FV3:-${NTHREADS_FCST:-${nth_fv3:-1}}} - NTHREADS_FV3=${nth_fv3:-1} cores_per_node=${cores_per_node:-${npe_node_fcst:-24}} ntiles=${ntiles:-6} if [ $MEMBER -lt 0 ]; then diff --git a/ush/rocoto/workflow_utils.py b/ush/rocoto/workflow_utils.py index 9296d07d894..e0011d00c87 100755 --- a/ush/rocoto/workflow_utils.py +++ b/ush/rocoto/workflow_utils.py @@ -259,7 +259,10 @@ def get_resources(machine, cfg, task, reservation, cdump='gdas'): ppn = cfg[f'npe_node_{ltask}'] if machine in [ 'WCOSS_DELL_P3', 'HERA', 'ORION', 'JET' ]: - threads = cfg[f'nth_{ltask}'] + if cdump in ['gfs'] and f'nth_{task}_gfs' in cfg.keys(): + threads = cfg[f'nth_{ltask}_gfs'] + else: + threads = cfg[f'nth_{ltask}'] nodes = np.int(np.ceil(np.float(tasks) / np.float(ppn)))