Skip to content

Fix separate threading for GFS and GDAS forecasts. #621

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
WalterKolczynski-NOAA merged 3 commits into from
Feb 9, 2022
Merged

Fix separate threading for GFS and GDAS forecasts. #621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4a6cb70
Fix separate threading for gfs and gdas forecasts.
DavidHuber-NOAA Jan 31, 2022
f1b9393
Cleaned up forecast_predet.sh, added threading info to WCOSS envs.
DavidHuber-NOAA Jan 31, 2022
cd62ddc
Added nth_fv3_gfs to env gfs if-block.
DavidHuber-NOAA Feb 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions env/HERA.env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,14 +130,18 @@ elif [ $step = "eupd" ]; then

elif [ $step = "fcst" ]; then

#PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs
if [[ $CDUMP == "gfs" ]]; then
npe_fcst=$npe_fcst_gfs
npe_node_fcst=$npe_node_fcst_gfs
nth_fv3=$nth_fv3_gfs
fi

nth_max=$(($npe_node_max / $npe_node_fcst))

export NTHREADS_FV3=${nth_fv3:-$nth_max}
[[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max
export cores_per_node=$npe_node_max
if [[ $CDUMP == "gfs" ]]; then
npe_fcst=$npe_fcst_gfs
fi
export APRUN_FV3="$launcher -n $npe_fcst"

export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
Expand Down
7 changes: 7 additions & 0 deletions env/JET.env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,13 @@ elif [ $step = "eupd" ]; then

elif [ $step = "fcst" ]; then

#PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs
if [[ $CDUMP == "gfs" ]]; then
npe_fcst=$npe_fcst_gfs
npe_node_fcst=$npe_node_fcst_gfs
nth_fv3=$nth_fv3_gfs
fi

nth_max=$(($npe_node_max / $npe_node_fcst))

export NTHREADS_FV3=${nth_fv3:-$nth_max}
Expand Down
10 changes: 7 additions & 3 deletions env/ORION.env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,14 +126,18 @@ elif [ $step = "eupd" ]; then

elif [ $step = "fcst" ]; then

#PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs
if [[ $CDUMP == "gfs" ]]; then
npe_fcst=$npe_fcst_gfs
npe_node_fcst=$npe_node_fcst_gfs
nth_fv3=$nth_fv3_gfs
fi

nth_max=$(($npe_node_max / $npe_node_fcst))

export NTHREADS_FV3=${nth_fv3:-$nth_max}
[[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max
export cores_per_node=$npe_node_max
if [[ $CDUMP == "gfs" ]]; then
npe_fcst=$npe_fcst_gfs
fi
export APRUN_FV3="$launcher -n $npe_fcst"

export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
Expand Down
13 changes: 8 additions & 5 deletions env/WCOSS_C.env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,17 +115,20 @@ elif [ $step = "eupd" ]; then

elif [ $step = "fcst" ]; then

#PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs
if [[ $CDUMP == "gfs" ]]; then
npe_fcst=$npe_fcst_gfs
npe_node_fcst=$npe_node_fcst_gfs
nth_fv3=$nth_fv3_gfs
fi

nth_max=$(($npe_node_max / $npe_node_fcst))

export NTHREADS_FV3=${nth_fv3:-$nth_max}
[[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max
export cores_per_node=$npe_node_max
#export APRUN_FV3="$launcher -j 1 -n ${npe_fv3:-$npe_fcst} -N $npe_node_fcst -d $NTHREADS_FV3 -cc depth"
if [ $CDUMP = "gdas" ]; then
export APRUN_FV3="$launcher -j 1 -n ${npe_fcst} -N $npe_node_fcst -d $NTHREADS_FV3 -cc depth"
else
export APRUN_FV3="$launcher -j 1 -n ${npe_fcst_gfs} -N $npe_node_fcst -d $NTHREADS_FV3 -cc depth"
fi
export APRUN_FV3="$launcher -j 1 -n ${npe_fcst} -N $npe_node_fcst -d $NTHREADS_FV3 -cc depth"

export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
[[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max
Expand Down
15 changes: 9 additions & 6 deletions env/WCOSS_DELL_P3.env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,17 +120,20 @@ elif [ $step = "eupd" ]; then

elif [ $step = "fcst" ]; then

#PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs
if [[ $CDUMP == "gfs" ]]; then
npe_fcst=$npe_fcst_gfs
npe_node_fcst=$npe_node_fcst_gfs
nth_fv3=$nth_fv3_gfs
fi

nth_max=$(($npe_node_max / $npe_node_fcst))

export NTHREADS_FV3=${nth_fv3:-$nth_max}
[[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max
export cores_per_node=$npe_node_max
if [ $CDUMP = "gdas" ]; then
#export APRUN_FV3="$launcher ${npe_fv3:-${npe_fcst:-$PBS_NP}}"
export APRUN_FV3="$launcher ${npe_fcst:-$PBS_NP}"
else
export APRUN_FV3="$launcher ${npe_fcst_gfs:-$PBS_NP}"
fi
#export APRUN_FV3="$launcher ${npe_fv3:-${npe_fcst:-$PBS_NP}}"
export APRUN_FV3="$launcher ${npe_fcst:-$PBS_NP}"
export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
[[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max
export APRUN_REGRID_NEMSIO="$launcher $LEVS"
Expand Down
1 change: 0 additions & 1 deletion parm/config/config.fcst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ echo "BEGIN: config.fcst"

# Source model specific information that is resolution dependent
. $EXPDIR/config.fv3 $CASE
[[ "$CDUMP" == "gfs" ]] && export nth_fv3=$nth_fv3_gfs

# Turn off waves if not used for this CDUMP
case $WAVE_CDUMP in
Expand Down
3 changes: 3 additions & 0 deletions parm/config/config.resources
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,8 +209,11 @@ elif [ $step = "fcst" ]; then
NTASKS_TOT=$ATMPETS

export nth_fcst=${nth_fv3:-2}
export nth_fcst_gfs=${nth_fv3_gfs:-2}

export npe_node_fcst=$(echo "$npe_node_max / $nth_fcst" | bc)
export npe_node_fcst_gfs=$(echo "$npe_node_max / $nth_fcst_gfs" | bc)

if [[ "$machine" == "WCOSS_C" ]]; then export memory_fcst="1024M"; fi

if [[ $DO_WAVE == "YES" ]]; then
Expand Down
1 change: 0 additions & 1 deletion ush/forecast_predet.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,6 @@ FV3_GFS_predet(){
#file and the value of npe_node_fcst is not correctly defined when using more than
#one thread and sets NTHREADS_FV3=1 even when the number of threads is appropraitely >1
#NTHREADS_FV3=${NTHREADS_FV3:-${NTHREADS_FCST:-${nth_fv3:-1}}}
NTHREADS_FV3=${nth_fv3:-1}
cores_per_node=${cores_per_node:-${npe_node_fcst:-24}}
ntiles=${ntiles:-6}
if [ $MEMBER -lt 0 ]; then
Expand Down
5 changes: 4 additions & 1 deletion ush/rocoto/workflow_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,10 @@ def get_resources(machine, cfg, task, reservation, cdump='gdas'):
ppn = cfg[f'npe_node_{ltask}']

if machine in [ 'WCOSS_DELL_P3', 'HERA', 'ORION', 'JET' ]:
threads = cfg[f'nth_{ltask}']
if cdump in ['gfs'] and f'nth_{task}_gfs' in cfg.keys():
threads = cfg[f'nth_{ltask}_gfs']
else:
threads = cfg[f'nth_{ltask}']

nodes = np.int(np.ceil(np.float(tasks) / np.float(ppn)))

Expand Down