diff --git a/env/HERA.env b/env/HERA.env index b7561883b53..45dd1d5265d 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -17,6 +17,7 @@ step=$1 export npe_node_max=40 export launcher="srun -l --export=ALL" +export mpmd_opt="--multi-prog" # Configure MPI environment #export I_MPI_ADJUST_ALLREDUCE=5 @@ -47,17 +48,16 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostbndpntbll" -o $step = "wavepostpnt" ]; then - export mpmd="--multi-prog" export CFP_MP="YES" if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi export wavempexec=${launcher} - export wave_mpmd=${mpmd} + export wave_mpmd=${mpmd_opt} elif [ $step = "atmanalrun" ]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -n \$ncmd --multi-prog" + export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}" nth_max=$(($npe_node_max / $npe_node_atmanalrun)) @@ -69,7 +69,7 @@ elif [ $step = "atmensanalrun" ]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -n \$ncmd --multi-prog" + export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}" nth_max=$(($npe_node_max / $npe_node_atmensanalrun)) @@ -79,7 +79,7 @@ elif [ $step = "atmensanalrun" ]; then elif [ $step = "aeroanlrun" ]; then - export APRUNCFP="$launcher -n \$ncmd --multi-prog" + export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}" nth_max=$(($npe_node_max / $npe_node_aeroanlrun)) @@ -94,17 +94,17 @@ elif [ $step = "anal" -o $step = "analcalc" ]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -n \$ncmd --multi-prog" + export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}" nth_max=$(($npe_node_max / $npe_node_anal)) export NTHREADS_GSI=${nth_anal:-$nth_max} [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher" + export APRUN_GSI="$launcher -n ${npe_gsi:-$npe_anal}" export NTHREADS_CALCINC=${nth_calcinc:-1} [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher" + export APRUN_CALCINC="$launcher \$ncmd" export NTHREADS_CYCLE=${nth_cycle:-12} [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max @@ -143,7 +143,7 @@ elif [ $step = "gldas" ]; then # periods being processed. npe_gldas_data_proc=$(($gldas_spinup_hours + 12)) - export APRUN_GLDAS_DATA_PROC="$launcher -n $npe_gldas_data_proc --multi-prog" + export APRUN_GLDAS_DATA_PROC="$launcher -n $npe_gldas_data_proc ${mpmd_opt}" elif [ $step = "eobs" ]; then @@ -154,11 +154,11 @@ elif [ $step = "eobs" ]; then export NTHREADS_GSI=${nth_eobs:-$nth_max} [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher" + export APRUN_GSI="$launcher -n ${npe_gsi:-$npe_eobs}" export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -n \$ncmd --multi-prog" + export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}" elif [ $step = "eupd" ]; then @@ -166,11 +166,11 @@ elif [ $step = "eupd" ]; then export NTHREADS_ENKF=${nth_eupd:-$nth_max} [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max - export APRUN_ENKF="$launcher" + export APRUN_ENKF="$launcher -n ${npe_enkf:-$npe_eupd}" export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -n \$ncmd --multi-prog" + export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}" elif [ $step = "fcst" ]; then @@ -190,11 +190,11 @@ elif [ $step = "fcst" ]; then export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher" + export APRUN_REGRID_NEMSIO="$launcher -n ${LEVS}" export NTHREADS_REMAP=${nth_remap:-2} [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max - export APRUN_REMAP="$launcher" + export APRUN_REMAP="$launcher -n ${npe_remap:-$npe_fcst}" export I_MPI_DAPL_UD="enable" elif [ $step = "efcs" ]; then @@ -208,7 +208,7 @@ elif [ $step = "efcs" ]; then export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" + export APRUN_REGRID_NEMSIO="$launcher -n ${LEVS}" elif [ $step = "post" ]; then @@ -216,11 +216,11 @@ elif [ $step = "post" ]; then export NTHREADS_NP=${nth_np:-1} [[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max - export APRUN_NP="$launcher" + export APRUN_NP="$launcher -n ${npe_post}" export NTHREADS_DWN=${nth_dwn:-1} [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max - export APRUN_DWN="$launcher" + export APRUN_DWN="$launcher -n ${npe_dwn}" elif [ $step = "ecen" ]; then @@ -228,7 +228,7 @@ elif [ $step = "ecen" ]; then export NTHREADS_ECEN=${nth_ecen:-$nth_max} [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max - export APRUN_ECEN="$launcher" + export APRUN_ECEN="$launcher -n ${npe_ecen}" export NTHREADS_CHGRES=${nth_chgres:-12} [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max @@ -236,7 +236,7 @@ elif [ $step = "ecen" ]; then export NTHREADS_CALCINC=${nth_calcinc:-1} [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher" + export APRUN_CALCINC="$launcher -n ${npe_ecen}" elif [ $step = "esfc" ]; then @@ -256,11 +256,11 @@ elif [ $step = "epos" ]; then export NTHREADS_EPOS=${nth_epos:-$nth_max} [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max - export APRUN_EPOS="$launcher" + export APRUN_EPOS="$launcher -n $npe_epos" elif [ $step = "init" ]; then - export APRUN="$launcher" + export APRUN="$launcher -n ${npe_init}" elif [ $step = "postsnd" ]; then @@ -268,11 +268,11 @@ elif [ $step = "postsnd" ]; then export NTHREADS_POSTSND=${nth_postsnd:-1} [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max - export APRUN_POSTSND="$launcher" + export APRUN_POSTSND="$launcher -n ${npe_postsnd}" export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max - export APRUN_POSTSNDCFP="$launcher" + export APRUN_POSTSNDCFP="$launcher -n ${npe_postsnd} ${mpmd_opt}" elif [ $step = "awips" ]; then @@ -280,13 +280,13 @@ elif [ $step = "awips" ]; then export NTHREADS_AWIPS=${nth_awips:-2} [[ $NTHREADS_AWIPS -gt $nth_max ]] && export NTHREADS_AWIPS=$nth_max - export APRUN_AWIPSCFP="$launcher -n $npe_awips --multi-prog" + export APRUN_AWIPSCFP="$launcher -n $npe_awips ${mpmd_opt}" elif [ $step = "gempak" ]; then nth_max=$(($npe_node_max / $npe_node_gempak)) export NTHREADS_GEMPAK=${nth_gempak:-1} - [[ $NTHREADS_GEMPAK -gt $nth_max ]] && export NTHREADS_GEMPAK=$nth_max - export APRUN="$launcher -n $npe_gempak --multi-prog" + [[ $NTHREADS_GEMPAK -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max} + export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}" fi diff --git a/env/ORION.env b/env/ORION.env index 04bae19e0e0..d8bc5fc6648 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -17,6 +17,7 @@ step=$1 export npe_node_max=40 export launcher="srun -l --export=ALL" +export mpmd_opt="--multi-prog" # Configure MPI environment export MPI_BUFS_PER_PROC=2048 @@ -46,17 +47,16 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostpnt" ]; then - export mpmd="--multi-prog" export CFP_MP="YES" if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi export wavempexec=${launcher} - export wave_mpmd=${mpmd} + export wave_mpmd=${mpmd_opt} elif [ $step = "atmanalrun" ]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -n \$ncmd --multi-prog" + export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}" nth_max=$(($npe_node_max / $npe_node_atmanalrun)) @@ -68,7 +68,7 @@ elif [ $step = "atmensanalrun" ]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -n \$ncmd --multi-prog" + export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}" nth_max=$(($npe_node_max / $npe_node_atmensanalrun)) @@ -78,7 +78,7 @@ elif [ $step = "atmensanalrun" ]; then elif [ $step = "aeroanlrun" ]; then - export APRUNCFP="$launcher -n \$ncmd --multi-prog" + export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}" nth_max=$(($npe_node_max / $npe_node_aeroanlrun)) @@ -93,17 +93,17 @@ elif [ $step = "anal" -o $step = "analcalc" ]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -n \$ncmd --multi-prog" + export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}" nth_max=$(($npe_node_max / $npe_node_anal)) export NTHREADS_GSI=${nth_anal:-$nth_max} [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher" + export APRUN_GSI="$launcher -n ${npe_gsi:-$npe_anal}" export NTHREADS_CALCINC=${nth_calcinc:-1} [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher" + export APRUN_CALCINC="$launcher \$ncmd" export NTHREADS_CYCLE=${nth_cycle:-12} [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max @@ -140,7 +140,7 @@ elif [ $step = "gldas" ]; then # periods being processed. npe_gldas_data_proc=$(($gldas_spinup_hours + 12)) - export APRUN_GLDAS_DATA_PROC="$launcher -n $npe_gldas_data_proc --multi-prog" + export APRUN_GLDAS_DATA_PROC="$launcher -n $npe_gldas_data_proc ${mpmd_opt}" elif [ $step = "eobs" ]; then @@ -149,25 +149,25 @@ elif [ $step = "eobs" ]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -n \$ncmd --multi-prog" + export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}" nth_max=$(($npe_node_max / $npe_node_eobs)) export NTHREADS_GSI=${nth_eobs:-$nth_max} [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher" + export APRUN_GSI="$launcher -n ${npe_gsi:-$npe_eobs}" elif [ $step = "eupd" ]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -n \$ncmd --multi-prog" + export APRUNCFP="$launcher -n \$ncmd ${mpmd_opt}" nth_max=$(($npe_node_max / $npe_node_eupd)) export NTHREADS_ENKF=${nth_eupd:-$nth_max} [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max - export APRUN_ENKF="$launcher" + export APRUN_ENKF="$launcher -n ${npe_enkf:-$npe_eupd}" elif [ $step = "fcst" ]; then @@ -187,11 +187,11 @@ elif [ $step = "fcst" ]; then export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher" + export APRUN_REGRID_NEMSIO="$launcher -n ${LEVS}" export NTHREADS_REMAP=${nth_remap:-2} [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max - export APRUN_REMAP="$launcher" + export APRUN_REMAP="$launcher -n ${npe_remap:-$npe_fcst}" export I_MPI_DAPL_UD="enable" elif [ $step = "efcs" ]; then @@ -205,7 +205,7 @@ elif [ $step = "efcs" ]; then export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" + export APRUN_REGRID_NEMSIO="$launcher -n ${LEVS}" elif [ $step = "post" ]; then @@ -213,11 +213,11 @@ elif [ $step = "post" ]; then export NTHREADS_NP=${nth_np:-1} [[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max - export APRUN_NP="$launcher" + export APRUN_NP="$launcher -n ${npe_post}" export NTHREADS_DWN=${nth_dwn:-1} [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max - export APRUN_DWN="$launcher" + export APRUN_DWN="$launcher -n ${npe_dwn}" elif [ $step = "ecen" ]; then @@ -225,7 +225,7 @@ elif [ $step = "ecen" ]; then export NTHREADS_ECEN=${nth_ecen:-$nth_max} [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max - export APRUN_ECEN="$launcher" + export APRUN_ECEN="$launcher -n $npe_ecen" export NTHREADS_CHGRES=${nth_chgres:-12} [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max @@ -233,7 +233,7 @@ elif [ $step = "ecen" ]; then export NTHREADS_CALCINC=${nth_calcinc:-1} [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher" + export APRUN_CALCINC="$launcher -n $npe_ecen" elif [ $step = "esfc" ]; then @@ -253,11 +253,11 @@ elif [ $step = "epos" ]; then export NTHREADS_EPOS=${nth_epos:-$nth_max} [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max - export APRUN_EPOS="$launcher" + export APRUN_EPOS="$launcher -n $npe_epos" elif [ $step = "init" ]; then - export APRUN="$launcher" + export APRUN="$launcher -n ${npe_init}" elif [ $step = "postsnd" ]; then @@ -265,11 +265,11 @@ elif [ $step = "postsnd" ]; then export NTHREADS_POSTSND=${nth_postsnd:-1} [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max - export APRUN_POSTSND="$launcher" + export APRUN_POSTSND="$launcher -n ${npe_postsnd}" export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max - export APRUN_POSTSNDCFP="$launcher" + export APRUN_POSTSNDCFP="$launcher -n ${npe_postsnd} ${mpmd_opt}" elif [ $step = "awips" ]; then @@ -277,13 +277,13 @@ elif [ $step = "awips" ]; then export NTHREADS_AWIPS=${nth_awips:-2} [[ $NTHREADS_AWIPS -gt $nth_max ]] && export NTHREADS_AWIPS=$nth_max - export APRUN_AWIPSCFP="$launcher -n $npe_awips --multi-prog" + export APRUN_AWIPSCFP="$launcher -n $npe_awips ${mpmd_opt}" elif [ $step = "gempak" ]; then - nth_max=$(($npe_node_max / $npe_node_gempak)) + nth_max=$(($npe_node_max / ${npe_node_gempak})) export NTHREADS_GEMPAK=${nth_gempak:-1} - [[ $NTHREADS_GEMPAK -gt $nth_max ]] && export NTHREADS_GEMPAK=$nth_max - export APRUN="$launcher -n $npe_gempak --multi-prog" + [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max} + export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}" fi diff --git a/parm/config/config.resources b/parm/config/config.resources index 7ee4d5305cf..bee8a83d466 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -598,14 +598,12 @@ elif [[ ${step} = "eobs" || ${step} = "eomg" ]]; then export wtime_eobs="00:15:00" export wtime_eomg="01:00:00" - if [ ${CASE} = "C768" ]; then + if [[ ${CASE} = "C768" ]]; then export npe_eobs=200 - elif [ ${CASE} = "C384" ]; then + elif [[ ${CASE} = "C384" ]]; then export npe_eobs=100 - elif [ ${CASE} = "C192" ]; then + elif [[ ${CASE} = "C192" || ${CASE} = "C96" || ${CASE} = "C48" ]]; then export npe_eobs=40 - elif [[ ${CASE} = "C96" || ${CASE} = "C48" ]]; then - export npe_eobs=20 fi export npe_eomg=${npe_eobs} export nth_eobs=2