diff --git a/env/JET.env b/env/JET.env deleted file mode 100755 index 5b14faf1a..000000000 --- a/env/JET.env +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/ksh -x - -if [ $# -ne 1 ]; then - - echo "Must specify an input argument to set runtime environment variables!" - echo "argument can be any one of the following:" - echo "anal fcst post vrfy" - echo "eobs eupd ecen efcs epos" - echo "postsnd awips gempak" - exit 1 - -fi - -step=$1 - -# Theia information -export npe_node_max=24 -export launcher="mpirun -np" - -# Configure MPI environment -export MPI_BUFS_PER_PROC=2048 -export MPI_BUFS_PER_HOST=2048 -export MPI_GROUP_MAX=256 -export MPI_MEMMAP_OFF=1 -export MP_STDOUTMODE="ORDERED" -export OMP_STACKSIZE=2048000 -export NTHSTACK=1024000000 -#export LD_BIND_NOW=1 - -if [ $step = "prep" -o $step = "prepbufr" ]; then - - nth_max=$(($npe_node_max / $npe_node_prep)) - - export POE="NO" - export BACK=${BACK:-"YES"} - export sys_tp="Cray-CS400" - -elif [ $step = "anal" ]; then - - nth_max=$(($npe_node_max / $npe_node_anal)) - - export NTHREADS_GSI=${nth_gsi:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher ${npe_gsi:-${npe_anal:-$PBS_NP}}" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher \$ncmd" - - export NTHREADS_CYCLE=${nth_cycle:-12} - [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max - npe_cycle=${ntiles:-6} - export APRUN_CYCLE="$launcher $npe_cycle" - - export NTHREADS_GAUSFCANL=1 - npe_gausfcanl=${npe_gausfcanl:-1} - export APRUN_GAUSFCANL="$launcher $npe_gausfcanl" - -elif [ $step = "eobs" ]; then - - nth_max=$(($npe_node_max / $npe_node_eobs)) - - export NTHREADS_GSI=${nth_gsi:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher ${npe_gsi:-${npe_eobs:-$PBS_NP}}" - -elif [ $step = "eupd" ]; then - - nth_max=$(($npe_node_max / $npe_node_eupd)) - - export NTHREADS_ENKF=${nth_enkf:-$nth_max} - [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max - export APRUN_ENKF="$launcher ${npe_enkf:-${npe_eupd:-$PBS_NP}}" - -elif [ $step = "fcst" ]; then - - nth_max=$(($npe_node_max / $npe_node_fcst)) - - export NTHREADS_FV3=${nth_fv3:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher ${npe_fv3:-${npe_fcst:-$PBS_NP}}" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" - - export NTHREADS_REMAP=${nth_remap:-2} - [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max - export APRUN_REMAP="$launcher ${npe_remap:-${npe_fcst:-$PBS_NP}}" - -elif [ $step = "efcs" ]; then - - nth_max=$(($npe_node_max / $npe_node_efcs)) - - export NTHREADS_FV3=${nth_fv3:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher ${npe_fv3:-${npe_efcs:-$PBS_NP}}" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" - -elif [ $step = "post" ]; then - - nth_max=$(($npe_node_max / $npe_node_post)) - - export NTHREADS_NP=${nth_np:-1} - [[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max - export APRUN_NP="$launcher ${npe_np:-${npe_post:-$PBS_NP}}" - - export NTHREADS_DWN=${nth_dwn:-1} - [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max - export APRUN_DWN="$launcher ${npe_dwn:-$PBS_NP}" - -elif [ $step = "ecen" ]; then - - nth_max=$(($npe_node_max / $npe_node_ecen)) - - export NTHREADS_ECEN=${nth_ecen:-$nth_max} - [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max - export APRUN_ECEN="$launcher ${npe_ecen:-$PBS_NP}" - - export NTHREADS_CHGRES=${nth_chgres:-12} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher ${npe_ecen:-$PBS_NP}" - -elif [ $step = "epos" ]; then - - nth_max=$(($npe_node_max / $npe_node_epos)) - - export NTHREADS_EPOS=${nth_epos:-$nth_max} - [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max - export APRUN_EPOS="$launcher ${npe_epos:-$PBS_NP}" - -elif [ $step = "fv3ic" ]; then - - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" - -elif [ $step = "postsnd" ]; then - - nth_max=$(($npe_node_max / $npe_node_postsnd)) - - export NTHREADS_POSTSND=${nth_postsnd:-1} - [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max - export APRUN_POSTSND="$launcher $npe_postsnd" - - export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} - [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max - export APRUN_POSTSNDCFP="$launcher $npe_postsndcfp" - -elif [ $step = "awips" ]; then - - echo "WARNING: $step is not enabled on $machine!" - -elif [ $step = "gempak" ]; then - - echo "WARNING: $step is not enabled on $machine!" - -fi diff --git a/env/THEIA.env b/env/THEIA.env deleted file mode 100755 index c747e1f99..000000000 --- a/env/THEIA.env +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/ksh -x - -if [ $# -ne 1 ]; then - - echo "Must specify an input argument to set runtime environment variables!" - echo "argument can be any one of the following:" - echo "anal fcst post vrfy" - echo "eobs eupd ecen efcs epos" - echo "postsnd awips gempak" - exit 1 - -fi - -step=$1 - -# Theia information -export npe_node_max=24 -export launcher="mpirun -np" - -# Configure MPI environment -export MPI_BUFS_PER_PROC=2048 -export MPI_BUFS_PER_HOST=2048 -export MPI_GROUP_MAX=256 -export MPI_MEMMAP_OFF=1 -export MP_STDOUTMODE="ORDERED" -export OMP_STACKSIZE=2048000 -export NTHSTACK=1024000000 -#export LD_BIND_NOW=1 - -if [ $step = "prep" -o $step = "prepbufr" ]; then - - nth_max=$(($npe_node_max / $npe_node_prep)) - - export POE="NO" - export BACK=${BACK:-"YES"} - export sys_tp="Cray-CS400" - -elif [ $step = "anal" ]; then - - nth_max=$(($npe_node_max / $npe_node_anal)) - - export NTHREADS_GSI=${nth_gsi:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher ${npe_gsi:-${npe_anal:-$PBS_NP}}" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher \$ncmd" - - export NTHREADS_CYCLE=${nth_cycle:-12} - [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max - npe_cycle=${ntiles:-6} - export APRUN_CYCLE="$launcher $npe_cycle" - - export NTHREADS_GAUSFCANL=1 - npe_gausfcanl=${npe_gausfcanl:-1} - export APRUN_GAUSFCANL="$launcher $npe_gausfcanl" - -elif [ $step = "eobs" ]; then - - nth_max=$(($npe_node_max / $npe_node_eobs)) - - export NTHREADS_GSI=${nth_gsi:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher ${npe_gsi:-${npe_eobs:-$PBS_NP}}" - -elif [ $step = "eupd" ]; then - - nth_max=$(($npe_node_max / $npe_node_eupd)) - - export NTHREADS_ENKF=${nth_enkf:-$nth_max} - [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max - export APRUN_ENKF="$launcher ${npe_enkf:-${npe_eupd:-$PBS_NP}}" - -elif [ $step = "fcst" ]; then - - nth_max=$(($npe_node_max / $npe_node_fcst)) - - export NTHREADS_FV3=${nth_fv3:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher ${npe_fv3:-${npe_fcst:-$PBS_NP}}" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" - - export NTHREADS_REMAP=${nth_remap:-2} - [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max - export APRUN_REMAP="$launcher ${npe_remap:-${npe_fcst:-$PBS_NP}}" - -elif [ $step = "efcs" ]; then - - nth_max=$(($npe_node_max / $npe_node_efcs)) - - export NTHREADS_FV3=${nth_fv3:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher ${npe_fv3:-${npe_efcs:-$PBS_NP}}" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" - -elif [ $step = "post" ]; then - - nth_max=$(($npe_node_max / $npe_node_post)) - - export NTHREADS_NP=${nth_np:-1} - [[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max - export APRUN_NP="$launcher ${npe_np:-${npe_post:-$PBS_NP}}" - - export NTHREADS_DWN=${nth_dwn:-1} - [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max - export APRUN_DWN="$launcher ${npe_dwn:-$PBS_NP}" - -elif [ $step = "ecen" ]; then - - nth_max=$(($npe_node_max / $npe_node_ecen)) - - export NTHREADS_ECEN=${nth_ecen:-$nth_max} - [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max - export APRUN_ECEN="$launcher ${npe_ecen:-$PBS_NP}" - - export NTHREADS_CHGRES=${nth_chgres:-12} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher ${npe_ecen:-$PBS_NP}" - -elif [ $step = "epos" ]; then - - nth_max=$(($npe_node_max / $npe_node_epos)) - - export NTHREADS_EPOS=${nth_epos:-$nth_max} - [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max - export APRUN_EPOS="$launcher ${npe_epos:-$PBS_NP}" - -elif [ $step = "fv3ic" ]; then - - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" - -elif [ $step = "postsnd" ]; then - - nth_max=$(($npe_node_max / $npe_node_postsnd)) - - export NTHREADS_POSTSND=${nth_postsnd:-1} - [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max - export APRUN_POSTSND="$launcher $npe_postsnd" - - export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} - [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max - export APRUN_POSTSNDCFP="$launcher $npe_postsndcfp cfp" - -elif [ $step = "awips" ]; then - - echo "WARNING: $step is not enabled on $machine!" - -elif [ $step = "gempak" ]; then - - echo "WARNING: $step is not enabled on $machine!" - -fi diff --git a/env/WCOSS_C.env b/env/WCOSS_C.env deleted file mode 100755 index 506f187d8..000000000 --- a/env/WCOSS_C.env +++ /dev/null @@ -1,194 +0,0 @@ -#!/bin/ksh -x - -if [ $# -ne 1 ]; then - - echo "Must specify an input argument to set runtime environment variables!" - echo "argument can be any one of the following:" - echo "anal fcst post vrfy" - echo "eobs eupd ecen efcs epos" - echo "postsnd awips gempak" - exit 1 - -fi - -step=$1 - -# Cray information -export launcher="aprun" -export npe_node_max=24 - -# Configure MPI environment -export MPI_BUFS_PER_PROC=2048 -export MPI_BUFS_PER_HOST=2048 -export MPI_GROUP_MAX=256 -export MPI_MEMMAP_OFF=1 -export MP_STDOUTMODE="ORDERED" -export NTHSTACK=1024000000 -export OMP_STACKSIZE="2048M" -export KMP_AFFINITY="disabled" - -if [ $step = "prep" -o $step = "prepbufr" ]; then - - nth_max=$(($npe_node_max / $npe_node_prep)) - - export POE=${POE:-"YES"} - export sys_tp="Cray-XC40" - -elif [ $step = "anal" ]; then - - nth_max=$(($npe_node_max / $npe_node_anal)) - - export NTHREADS_GSI=${nth_gsi:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher -j 1 -n ${npe_gsi:-$npe_anal} -N $npe_node_anal -d $NTHREADS_GSI -cc depth" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher -j 1 -n \$ncmd -N 1 -d $NTHREADS_CALCINC -cc depth" - - if [ ${USE_CFP:-"NO"} = "YES" ]; then - export APRUNCFP="$launcher -q -j 1 -n \$ncmd -N \$ncmd_max -d 1 cfp" - fi - - export NTHREADS_CYCLE=${nth_cycle:-12} - [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max - npe_node_cycle=$(($npe_node_max / $NTHREADS_CYCLE)) - npe_cycle=${ntiles:-6} - [[ $npe_node_cycle -gt $npe_cycle ]] && npe_node_cycle=$npe_cycle - export APRUN_CYCLE="$launcher -j 1 -n $npe_cycle -N $npe_node_cycle -d $NTHREADS_CYCLE -cc depth" - - export NTHREADS_GAUSFCANL=1 - npe_node_gausfcanl=$(($npe_node_max / $NTHREADS_GAUSFCANL)) - npe_gausfcanl=${npe_gausfcanl:-1} - [[ $npe_node_gausfcanl -gt $npe_gausfcanl ]] && npe_node_gausfcanl=$npe_gausfcanl - export APRUN_GAUSFCANL="$launcher -j 1 -n $npe_gausfcanl -N $npe_node_gausfcanl -d $NTHREADS_GAUSFCANL -cc depth" - -elif [ $step = "eobs" ]; then - - nth_max=$(($npe_node_max / $npe_node_eobs)) - - export NTHREADS_GSI=${nth_gsi:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher -j 1 -n ${npe_gsi:-$npe_eobs} -N $npe_node_eobs -d $NTHREADS_GSI -cc depth" - - if [ ${USE_CFP:-"NO"} = "YES" ]; then - export APRUNCFP="$launcher -q -j 1 -n \$ncmd -N \$ncmd_max -d 1 cfp" - fi - -elif [ $step = "eupd" ]; then - - nth_max=$(($npe_node_max / $npe_node_eupd)) - - export NTHREADS_ENKF=${nth_enkf:-$nth_max} - [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max - export APRUN_ENKF="$launcher -j 1 -n ${npe_enkf:-$npe_eupd} -N $npe_node_eupd -d $NTHREADS_ENKF -cc depth" - - if [ ${USE_CFP:-"NO"} = "YES" ]; then - export APRUNCFP="$launcher -q -j 1 -n \$ncmd -N \$ncmd_max -d 1 cfp" - fi - -elif [ $step = "fcst" ]; then - - nth_max=$(($npe_node_max / $npe_node_fcst)) - - export NTHREADS_FV3=${nth_fv3:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher -j 1 -n ${npe_fv3:-$npe_fcst} -N $npe_node_fcst -d $NTHREADS_FV3 -cc depth" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher -j 1 -n $LEVS -N $npe_node_fcst -d $NTHREADS_REGRID_NEMSIO -cc depth" - - export NTHREADS_REMAP=${nth_remap:-2} - [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max - export APRUN_REMAP="$launcher -j 1 -n ${npe_remap:-$npe_fcst} -N $npe_node_fcst -d $NTHREADS_REMAP -cc depth" - -elif [ $step = "efcs" ]; then - - nth_max=$(($npe_node_max / $npe_node_efcs)) - - export NTHREADS_FV3=${nth_fv3:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher -j 1 -n ${npe_fv3:-$npe_efcs} -N $npe_node_efcs -d $NTHREADS_FV3 -cc depth" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher -j 1 -n $LEVS -N $npe_node_efcs -d $NTHREADS_REGRID_NEMSIO -cc depth" - -elif [ $step = "post" ]; then - - nth_max=$(($npe_node_max / $npe_node_post)) - - export NTHREADS_NP=${nth_np:-1} - [[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max - export APRUN_NP="$launcher -j 1 -n ${npe_np:-$npe_post} -N $npe_node_post -d $NTHREADS_NP -cc depth" - - export NTHREADS_DWN=${nth_dwn:-1} - [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max - export APRUN_DWN="$launcher -j 1 -n $npe_dwn -N $npe_node_dwn -d $NTHREADS_DWN cfp" - -elif [ $step = "ecen" ]; then - - nth_max=$(($npe_node_max / $npe_node_ecen)) - - export NTHREADS_ECEN=${nth_ecen:-$nth_max} - [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max - export APRUN_ECEN="$launcher -j 1 -n $npe_ecen -N $npe_node_ecen -d $NTHREADS_ECEN -cc depth" - - export NTHREADS_CHGRES=${nth_chgres:-12} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="$launcher -j 1 -n 1 -N 1 -d $NTHREADS_CHGRES -cc depth" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher -j 1 -n $npe_ecen -N $npe_node_ecen -d $NTHREADS_CALCINC -cc depth" - -elif [ $step = "epos" ]; then - - nth_max=$(($npe_node_max / $npe_node_epos)) - - export NTHREADS_EPOS=${nth_epos:-$nth_max} - [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max - export APRUN_EPOS="$launcher -j 1 -n $npe_epos -N $npe_node_epos -d $NTHREADS_EPOS -cc depth" - -elif [ $step = "fv3ic" ]; then - - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="$launcher -j 1 -n 1 -N 1 -d $NTHREADS_CHGRES -cc depth" - -elif [ $step = "vrfy" ]; then - - export IOBUF_PARAMS="*:size=32M:count=4:verbose" - export APRUNTRACK="$launcher -j1 -n1 -N1 -d1 -cc depth" - -elif [ $step = "postsnd" ]; then - export IOBUF_PARAMS="sigf*:size=128M:count=20:prefetch=0:verbose,gfs_collectiv*:size=128M:count=2:prefetch=0:verbose,*.snd:size=128M:count=3:prefetch=0:verbose,*.sfc:size=32M:count=3:prefetch=0:verbose,bufr.*:size=8M:count=20:prefetch=0:verbose" - - nth_max=$(($npe_node_max / $npe_node_postsnd)) - - export NTHREADS_POSTSND=${nth_postsnd:-1} - [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max - export APRUN_POSTSND="$launcher -j 1 -n $npe_postsnd -N $npe_node_postsnd -d $NTHREADS_POSTSND -cc depth" - - export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} - [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max - export APRUN_POSTSNDCFP="$launcher -j 1 -n $npe_postsndcfp -N $npe_node_postsndcfp -d $NTHREADS_POSTSNDCFP cfp" - -elif [ $step = "awips" ]; then - nth_max=$(($npe_node_max / $npe_node_awips)) - - export NTHREADS_AWIPS=${nth_awips:-2} - [[ $NTHREADS_AWIPS -gt $nth_max ]] && export NTHREADS_AWIPS=$nth_max - export APRUN_AWIPSCFP="$launcher -j 1 -n $npe_awips -N $npe_node_awips -d $NTHREADS_AWIPS -cc depth cfp" - -elif [ $step = "gempak" ]; then - nth_max=$(($npe_node_max / $npe_node_gempak)) - - export NTHREADS_GEMPAK=${nth_gempak:-3} - [[ $NTHREADS_GEMPAK -gt $nth_max ]] && export NTHREADS_GEMPAK=$nth_max - export APRUN_GEMPAKCFP="$launcher -j 1 -n $npe_gempak -N $npe_node_gempak -d $NTHREADS_GEMPAK cfp" - -fi diff --git a/env/WCOSS_DELL_P3.env b/env/WCOSS_DELL_P3.env deleted file mode 100755 index 481fdb2a7..000000000 --- a/env/WCOSS_DELL_P3.env +++ /dev/null @@ -1,200 +0,0 @@ -#!/bin/ksh -x - -if [ $# -ne 1 ]; then - - echo "Must specify an input argument to set runtime environment variables!" - echo "argument can be any one of the following:" - echo "anal fcst post vrfy" - echo "eobs eupd ecen efcs epos" - echo "postsnd awips gempak" - exit 1 - -fi - -step=$1 - -# WCOSS_DELL_P3 information -export npe_node_max=28 -export launcher="mpirun -n" - -# Due to ESMF issue, fv3gfs model must run with npe_node_max=24 -if [ $step = "fcst" -o $step = "efcs" ]; then - export npe_node_max=24 -fi - -# Configure MPI environment -#export MPI_BUFS_PER_PROC=2048 -#export MPI_BUFS_PER_HOST=2048 -#export MPI_GROUP_MAX=256 -#export MPI_MEMMAP_OFF=1 -export MPI_LABELIO=YES -export MP_STDOUTMODE="ORDERED" -export KMP_STACKSIZE=2048M -export KMP_AFFINITY=scatter - -# get binding information -#export I_MPI_DEBUG=4 - -if [ $step = "prep" -o $step = "prepbufr" ]; then - - nth_max=$(($npe_node_max / $npe_node_prep)) - - export POE="NO" - export BACK=${BACK:-"YES"} - export sys_tp="Dell-p3" - -elif [ $step = "anal" ]; then - - nth_max=$(($npe_node_max / $npe_node_anal)) - - export NTHREADS_GSI=${nth_anal:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher ${npe_gsi:-${npe_anal:-$PBS_NP}}" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher \$ncmd" - - export NTHREADS_CYCLE=${nth_cycle:-14} - [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max - npe_cycle=${ntiles:-6} - export APRUN_CYCLE="$launcher $npe_cycle" - - export NTHREADS_GAUSFCANL=1 - npe_gausfcanl=${npe_gausfcanl:-1} - export APRUN_GAUSFCANL="$launcher $npe_gausfcanl" - - if [ ${USE_CFP:-"NO"} = "YES" ]; then - export APRUNCFP="$launcher \$ncmd cfp" - fi - -elif [ $step = "eobs" ]; then - - nth_max=$(($npe_node_max / $npe_node_eobs)) - - export NTHREADS_GSI=${nth_eobs:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher ${npe_gsi:-${npe_eobs:-$PBS_NP}}" - - if [ ${USE_CFP:-"NO"} = "YES" ]; then - export APRUNCFP="$launcher \$ncmd cfp" - fi - -elif [ $step = "eupd" ]; then - - nth_max=$(($npe_node_max / $npe_node_eupd)) - - export NTHREADS_ENKF=${nth_eupd:-$nth_max} - [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max - export APRUN_ENKF="$launcher ${npe_enkf:-${npe_eupd:-$PBS_NP}}" - - if [ ${USE_CFP:-"NO"} = "YES" ]; then - export APRUNCFP="$launcher \$ncmd cfp" - fi - -elif [ $step = "fcst" ]; then - - nth_max=$(($npe_node_max / $npe_node_fcst)) - - export NTHREADS_FV3=${nth_fv3:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher ${npe_fv3:-${npe_fcst:-$PBS_NP}}" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" - - export NTHREADS_REMAP=${nth_remap:-2} - [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max - export APRUN_REMAP="$launcher ${npe_remap:-${npe_fcst:-$PBS_NP}}" - -elif [ $step = "efcs" ]; then - - nth_max=$(($npe_node_max / $npe_node_efcs)) - - export NTHREADS_FV3=${nth_efcs:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher ${npe_fv3:-${npe_efcs:-$PBS_NP}}" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" - -elif [ $step = "post" ]; then - - nth_max=$(($npe_node_max / $npe_node_post)) - - export NTHREADS_NP=${nth_np:-1} - [[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max - export APRUN_NP="$launcher ${npe_np:-${npe_post:-$PBS_NP}}" - - export NTHREADS_DWN=${nth_dwn:-1} - [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max - export APRUN_DWN="$launcher ${npe_dwn:-$PBS_NP} cfp" - -elif [ $step = "ecen" ]; then - - nth_max=$(($npe_node_max / $npe_node_ecen)) - - export NTHREADS_ECEN=${nth_ecen:-$nth_max} - [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max - export APRUN_ECEN="$launcher ${npe_ecen:-$PBS_NP}" - - export NTHREADS_CHGRES=${nth_chgres:-14} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher ${npe_ecen:-$PBS_NP}" - - export NTHREADS_CYCLE=${nth_cycle:-14} - [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max - export APRUN_CYCLE="$launcher $npe_ecen" - -elif [ $step = "epos" ]; then - - nth_max=$(($npe_node_max / $npe_node_epos)) - - export NTHREADS_EPOS=${nth_epos:-$nth_max} - [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max - export APRUN_EPOS="$launcher ${npe_epos:-$PBS_NP}" - -elif [ $step = "fv3ic" ]; then - - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" - -elif [ $step = "postsnd" ]; then - - nth_max=$(($npe_node_max / $npe_node_postsnd)) - - export NTHREADS_POSTSND=${nth_postsnd:-1} - [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max - export APRUN_POSTSND="$launcher $npe_postsnd" - - export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} - [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max - export APRUN_POSTSNDCFP="$launcher $npe_postsndcfp cfp" - -elif [ $step = "awips" ]; then - - nth_max=$(($npe_node_max / $npe_node_awips)) - - export NTHREADS_AWIPS=${nth_awips:-2} - [[ $NTHREADS_AWIPS -gt $nth_max ]] && export NTHREADS_AWIPS=$nth_max - export APRUN_AWIPSCFP="$launcher ${npe_awips:-$PBS_NP} cfp" - -elif [ $step = "gempak" ]; then - - nth_max=$(($npe_node_max / $npe_node_gempak)) - - export NTHREADS_GEMPAK=${nth_gempak:-1} - [[ $NTHREADS_GEMPAK -gt $nth_max ]] && export NTHREADS_GEMPAK=$nth_max - export APRUN_GEMPAKCFP="$launcher \$ntasks cfp" - - -fi diff --git a/env/gfs.ver b/env/gfs.ver deleted file mode 100644 index efd4a1d9c..000000000 --- a/env/gfs.ver +++ /dev/null @@ -1,22 +0,0 @@ -export gfs_ver=v15.0.0 - -export crtm_ver=2.3.0 -export hwrf_ver=v11.0.0 -export g2tmpl_ver=1.4.0 - -export grib_util_ver=1.1.0 -export util_shared_ver=1.0.6 -export cfp_intel_sandybridge_ver=1.1.0 -export iobuf_ver=2.0.7 -export ESMF_intel_sandybridge_ver=3_1_0rp5 -export ESMF_intel_haswell_ver=3_1_0rp5 -export gempak_ver=7.3.0 -export old_gempak_ver=6.32.0 -export NCL_gnu_sandybridge_ver=6.3.0 -export ncarg_intel_sandybridge_ver=6.1.0 -export dumpjb_ver=4.0.0 - -## FOLLOWING are used by JGDAS_TROPC -export obsproc_dump_ver=v4.0.0 -export obsproc_shared_bufr_dumplist_ver=v1.5.0 - diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 06d5ab53e..639fd6c3b 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -124,7 +124,7 @@ case $MACHINE in ulimit -s unlimited ulimit -a APRUN="srun" - OMP_NUM_THREADS=4 + OMP_NUM_THREADS=2 ;; "ODIN") diff --git a/tests/baseline_configs/config.nco_CONUS_25km_GFDLgrid.sh b/tests/baseline_configs/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta.sh similarity index 100% rename from tests/baseline_configs/config.nco_CONUS_25km_GFDLgrid.sh rename to tests/baseline_configs/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta.sh diff --git a/tests/baseline_configs/config.nco_RRFS_CONUS_25km_HRRR_RAP.sh b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh similarity index 100% rename from tests/baseline_configs/config.nco_RRFS_CONUS_25km_HRRR_RAP.sh rename to tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh diff --git a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh new file mode 100644 index 000000000..c16cf5488 --- /dev/null +++ b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -0,0 +1,18 @@ +RUN_ENVIR="nco" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_HRRR" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200208" +DATE_LAST_CYCL="20200208" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.nco_RRFS_CONUS_3km_FV3GFS_FV3GFS.sh b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh similarity index 100% rename from tests/baseline_configs/config.nco_RRFS_CONUS_3km_FV3GFS_FV3GFS.sh rename to tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh diff --git a/tests/baseline_configs/config.nco_RRFS_CONUS_3km_HRRR_RAP.sh b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh similarity index 100% rename from tests/baseline_configs/config.nco_RRFS_CONUS_3km_HRRR_RAP.sh rename to tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh diff --git a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh new file mode 100644 index 000000000..88d007451 --- /dev/null +++ b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -0,0 +1,18 @@ +RUN_ENVIR="nco" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_3km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_HRRR" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200208" +DATE_LAST_CYCL="20200208" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.nco_RRFS_SUBCONUS_3km_HRRR_RAP.sh b/tests/baseline_configs/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh similarity index 100% rename from tests/baseline_configs/config.nco_RRFS_SUBCONUS_3km_HRRR_RAP.sh rename to tests/baseline_configs/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index 819833ed8..aa4e72a35 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -45,12 +45,14 @@ grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_SUBCONUS_3km_FV3GFS_FV3GFS grid_RRFS_SUBCONUS_3km_HRRR_RAP -nco_CONUS_25km_GFDLgrid -nco_RRFS_CONUS_25km_HRRR_RAP -nco_RRFS_CONUS_3km_FV3GFS_FV3GFS -nco_RRFS_CONUS_3km_HRRR_RAP -nco_RRFS_SUBCONUS_3km_HRRR_RAP nco_ensemble +nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta +nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR +nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR +nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional +nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR +nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR +nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR new_ESGgrid new_GFDLgrid new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE diff --git a/ush/config.sh.RRFS_AK_dev1 b/ush/config.sh.RRFS_AK_dev1 index b41af8bb9..3d4b8a13d 100644 --- a/ush/config.sh.RRFS_AK_dev1 +++ b/ush/config.sh.RRFS_AK_dev1 @@ -1,10 +1,19 @@ MACHINE="jet" ACCOUNT="nrtrr" +RESERVATION="rrfsdet" EXPT_BASEDIR="/home/rtrr/RRFS" EXPT_SUBDIR="RRFS_AK_dev1" +if [[ -n $RESERVATION ]] ; then + ACCOUNT=rtrr + PARTITION_DEFAULT=kjet + PARTITION_FCST=kjet + PARTITION_GRAPHICS=kjet +fi + FIXLAM_NCO_BASEDIR=/mnt/lfs4/BMC/nrtrr/RRFS/fix/fix_lam.20210128 FIXgsm=/mnt/lfs4/BMC/nrtrr/RRFS/fix/fix_am.20210128 +GWD_HRRRsuite_BASEDIR=/mnt/lfs4/BMC/nrtrr/RRFS/fix/fix_lam.20210128 VERBOSE="TRUE" @@ -14,7 +23,7 @@ PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME=RRFS_AK_3km QUILTING="TRUE" -CCPP_PHYS_SUITE="FV3_GSD_SAR" +CCPP_PHYS_SUITE="FV3_HRRR" FCST_LEN_HRS="36" LBC_SPEC_INTVL_HRS="6" @@ -33,6 +42,7 @@ EXTRN_MDL_NAME_LBCS="RAP" envir="para" NET="RRFS_AK" +TAG="RRFS_dev1_AK" ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ak_dev1" NCARG_ROOT="/apps/ncl/6.5.0-CentOS6.10_64bit_nodap_gnu447" diff --git a/ush/config.sh.RRFS_dev1 b/ush/config.sh.RRFS_dev1 index 5497095fc..3d5755144 100644 --- a/ush/config.sh.RRFS_dev1 +++ b/ush/config.sh.RRFS_dev1 @@ -1,10 +1,19 @@ MACHINE="jet" ACCOUNT="nrtrr" +RESERVATION="rrfsdet" EXPT_BASEDIR="/home/rtrr/RRFS" EXPT_SUBDIR="RRFS_dev1" +if [[ -n $RESERVATION ]] ; then + ACCOUNT=rtrr + PARTITION_DEFAULT=kjet + PARTITION_FCST=kjet + PARTITION_GRAPHICS=kjet +fi + FIXLAM_NCO_BASEDIR=/mnt/lfs4/BMC/nrtrr/RRFS/fix/fix_lam.20210128 FIXgsm=/mnt/lfs4/BMC/nrtrr/RRFS/fix/fix_am.20210128 +GWD_HRRRsuite_BASEDIR=/mnt/lfs4/BMC/nrtrr/RRFS/fix/fix_lam.20210128 VERBOSE="TRUE" @@ -14,7 +23,7 @@ PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME=RRFS_CONUS_3km QUILTING="TRUE" -CCPP_PHYS_SUITE="FV3_GSD_SAR" +CCPP_PHYS_SUITE="FV3_HRRR" FCST_LEN_HRS="36" LBC_SPEC_INTVL_HRS="6" @@ -32,6 +41,7 @@ EXTRN_MDL_NAME_LBCS="RAP" envir="para" NET="RRFS_CONUS" +TAG="RRFS_dev1_CONUS" ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_dev1" NCARG_ROOT="/apps/ncl/6.5.0-CentOS6.10_64bit_nodap_gnu447" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 21663b003..31eda88b4 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -103,6 +103,7 @@ RUN_ENVIR="nco" # MACHINE="BIG_COMPUTER" ACCOUNT="project_name" +RESERVATION="" SCHED="" PARTITION_DEFAULT="" QUEUE_DEFAULT="" @@ -110,6 +111,8 @@ PARTITION_HPSS="" QUEUE_HPSS="" PARTITION_FCST="" QUEUE_FCST="" +PARTITION_GRAPHICS="" +QUEUE_GRAPHICS="" # #----------------------------------------------------------------------- # @@ -186,7 +189,7 @@ EXPT_SUBDIR="" # that will point to a subdirectory (having the name of the grid being # used) under this directory. This variable should be set to a null # string in this file, but it can be specified in the user-specified -# workflow configuration file (EXPT_CONFIG_FN) +# workflow configuration file (EXPT_CONFIG_FN). # # STMP: # The beginning portion of the directory that will contain cycle-dependent @@ -232,6 +235,7 @@ STMP="/base/path/of/directory/containing/model/input/and/raw/output/files" NET="rrfs" envir="para" RUN="experiment_name" +TAG="dev_grid" PTMP="/base/path/of/directory/containing/postprocessed/output/files" ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_dev1" diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 7e6707f56..37705ba1c 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -163,6 +163,7 @@ settings="\ # Parameters needed by the job scheduler. # 'account': $ACCOUNT + 'reservation': $RESERVATION 'sched': $SCHED 'partition_default': ${PARTITION_DEFAULT} 'queue_default': ${QUEUE_DEFAULT} @@ -170,6 +171,8 @@ settings="\ 'queue_hpss': ${QUEUE_HPSS} 'partition_fcst': ${PARTITION_FCST} 'queue_fcst': ${QUEUE_FCST} + 'partition_graphics': ${PARTITION_GRAPHICS} + 'queue_graphics': ${QUEUE_GRAPHICS} 'machine': ${MACHINE} # # Workflow task names. @@ -183,6 +186,7 @@ settings="\ 'make_lbcs_tn': ${MAKE_LBCS_TN} 'run_fcst_tn': ${RUN_FCST_TN} 'run_post_tn': ${RUN_POST_TN} + 'tag': ${TAG} # # Number of nodes to use for each task. # @@ -199,7 +203,7 @@ settings="\ # Number of cores used for a task # 'ncores_run_fcst': ${PE_MEMBER01} - 'native_run_fcst': --cpus-per-task 4 --exclusive + 'native_run_fcst': --cpus-per-task 2 --exclusive # # Number of logical processes per node for each task. If running without # threading, this is equal to the number of MPI processes per node. diff --git a/ush/setup.sh b/ush/setup.sh index aab89d32e..1dc7693b5 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -406,6 +406,8 @@ case $MACHINE in QUEUE_HPSS=${QUEUE_HPSS:-"batch"} PARTITION_FCST=${PARTITION_FCST:-"sjet,vjet,kjet,xjet"} QUEUE_FCST=${QUEUE_FCST:-"batch"} + PARTITION_GRAPHICS=${PARTITION_GRAPHICS:-"kjet,xjet"} + QUEUE_GRAPHICS=${QUEUE_GRAPHICS:-"batch"} ;; "ODIN") @@ -1005,8 +1007,54 @@ fi # #----------------------------------------------------------------------- # -GWD_HRRRsuite_DIR="${GWD_HRRRsuite_BASEDIR}/${PREDEF_GRID_NAME}" +GWD_HRRRsuite_DIR="" if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then +# +# If in NCO mode, make sure that GWD_HRRRsuite_BASEDIR is set equal to +# FIXLAM_NCO_BASEDIR +# + if [ "${RUN_ENVIR}" = "nco" ]; then + + if [ "${GWD_HRRRsuite_BASEDIR}" != "${FIXLAM_NCO_BASEDIR}" ]; then + + gwd_hrrrsuite_basedir_orig="${GWD_HRRRsuite_BASEDIR}" + GWD_HRRRsuite_BASEDIR="${FIXLAM_NCO_BASEDIR}" + + if [ ! -z "${gwd_hrrrsuite_basedir_orig}" ]; then + print_err_msg_exit " +When RUN_ENVIR is set to \"nco\", the workflow assumes that the base +directory (GWD_HRRRsuite_BASEDIR) under which the grid-specific +subdirectories containing the gravity wave drag-related orography +statistics files for the FV3_HRRR suite are located is the same as the +base directory (FIXLAM_NCO_BASEDIR) under which the other fixed files +are located. Currently, this is not the case: + GWD_HRRRsuite_BASEDIR = \"${gwd_hrrrsuite_basedir_orig}\" + FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR}\" +Resetting GWD_HRRRsuite_BASEDIR to FIXLAM_NCO_BASEDIR. Reset value is: + GWD_HRRRsuite_BASEDIR = \"${GWD_HRRRsuite_BASEDIR}\"" + fi + + fi + + fi +# +# Check that GWD_HRRRsuite_BASEDIR exists and is a directory. +# + if [ ! -d "${GWD_HRRRsuite_BASEDIR}" ]; then + print_err_msg_exit "\ +The base directory (GWD_HRRRsuite_BASEDIR) under which the grid-specific +subdirectories containing the gravity wave drag-related orography files +for the FV3_HRRR suite should be located does not exist (or is not a +directory): + GWD_HRRRsuite_BASEDIR = \"${GWD_HRRRsuite_BASEDIR}\"" + fi + GWD_HRRRsuite_DIR="${GWD_HRRRsuite_BASEDIR}/${PREDEF_GRID_NAME}" +# +# Ensure that PREDEF_GRID_NAME is not set to a null string. Currently, +# only predefined grids can be used with the FV3_HRRR suite because +# orography statistics files required by this suite are available only +# for (some of) the predefined grids. +# if [ -z "${PREDEF_GRID_NAME}" ]; then print_err_msg_exit "\ A predefined grid name (PREDEF_GRID_NAME) must be specified when using @@ -1014,6 +1062,11 @@ the FV3_HRRR physics suite: CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\" PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\"" else +# +# Ensure that the directory GWD_HRRRsuite_DIR in which the orography +# statistics files required by the FV3_HRRR suite are located actually +# exists. +# if [ ! -d "${GWD_HRRRsuite_DIR}" ]; then print_err_msg_exit "\ The directory (GWD_HRRRsuite_DIR) that should contain the gravity wave @@ -1026,6 +1079,7 @@ drag related orography files for the FV3_HRRR suite is empty: GWD_HRRRsuite_DIR = \"${GWD_HRRRsuite_DIR}\"" fi fi + fi # #----------------------------------------------------------------------- diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index 749a68662..4f22d5182 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -16,6 +16,7 @@ FV3_GSD_v0: FV3_GSD_SAR: fv_core_nml: + fv_sg_adj: 0 hord_dp: -5 hord_mt: 5 hord_tm: 5 @@ -52,6 +53,7 @@ FV3_HRRR: kord_tm: -9 kord_tr: 9 kord_wz: 9 + nord_tr: 2 nrows_blend: 10 regional_bcs_from_gsi: false write_restart_with_bcs: false @@ -59,13 +61,16 @@ FV3_HRRR: <<: *gsd_sar_phys cdmbgwd: [3.5, 1.0] do_mynnsfclay: true + do_sfcperts: !!python/none effr_in: true gwd_opt: 3 + iaer: 5111 + icliq_sw: 2 imfdeepcnv: -1 imfshalcnv: -1 - lsoil: 9 + iovr: 3 -FV3_RRFS_v1alpha: +FV3_RRFS_v1beta: gfs_physics_nml: <<: *gsd_sar_phys do_deep: False diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 60e45a011..e59873d6e 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -13,11 +13,13 @@ See README.xml_templating.md for information on using the Templating mechanisms. - - - - - + + + + + + + {%- if partition_default is not none %} -&ACCOUNT;&QUEUE_DEFAULT;{{ partition_default }}"> +&ACCOUNT;&QUEUE_DEFAULT;{{ partition_default }}&RRFS_RESERVE;"> {%- else %} &ACCOUNT;&QUEUE_DEFAULT;"> {%- endif %} @@ -85,7 +88,7 @@ tasks; and the "FCST" type is used for the RUN_FCST_TN task. {%- else %} &ACCOUNT;&QUEUE_FCST;"> {%- endif %} -&ACCOUNT;&QUEUE_FCST;kjet,xjet"> +&ACCOUNT;&QUEUE_GRAPHICS;{{ partition_graphics }}"> @@ -155,7 +158,7 @@ MODULES_RUN_TASK_FP script. {% endif %} {{ wtime_make_grid }} &NCORES_PER_NODE; - &MAKE_GRID_TN; + &TAG;_&MAKE_GRID_TN; &LOGDIR;/&MAKE_GRID_TN;.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -179,7 +182,7 @@ MODULES_RUN_TASK_FP script. {% endif %} {{ wtime_make_orog }} &NCORES_PER_NODE; - &MAKE_OROG_TN; + &TAG;_&MAKE_OROG_TN; &LOGDIR;/&MAKE_OROG_TN;.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -207,7 +210,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_make_sfc_climo }}:ppn={{ ppn_make_sfc_climo }} {{ wtime_make_sfc_climo }} &NCORES_PER_NODE; - &MAKE_SFC_CLIMO_TN; + &TAG;_&MAKE_SFC_CLIMO_TN; &LOGDIR;/&MAKE_SFC_CLIMO_TN;.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -248,7 +251,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_get_extrn_ics }}:ppn={{ ppn_get_extrn_ics }} {{ wtime_get_extrn_ics }} &NCORES_PER_NODE; - &GET_EXTRN_ICS_TN; + &TAG;_&GET_EXTRN_ICS_TN; &LOGDIR;/&GET_EXTRN_ICS_TN;_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -281,7 +284,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_get_extrn_lbcs }}:ppn={{ ppn_get_extrn_lbcs }} {{ wtime_get_extrn_lbcs }} &NCORES_PER_NODE; - &GET_EXTRN_LBCS_TN; + &TAG;_&GET_EXTRN_LBCS_TN; &LOGDIR;/&GET_EXTRN_LBCS_TN;_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -325,7 +328,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_make_ics }}:ppn={{ ppn_make_ics }} {{ wtime_make_ics }} &NCORES_PER_NODE; - &MAKE_ICS_TN;{{ uscore_ensmem_name }} + &TAG;_&MAKE_ICS_TN;{{ uscore_ensmem_name }} &LOGDIR;/&MAKE_ICS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -369,7 +372,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_make_lbcs }}:ppn={{ ppn_make_lbcs }} {{ wtime_make_lbcs }} &NCORES_PER_NODE; - &MAKE_LBCS_TN;{{ uscore_ensmem_name }} + &TAG;_&MAKE_LBCS_TN;{{ uscore_ensmem_name }} &LOGDIR;/&MAKE_LBCS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -412,13 +415,13 @@ MODULES_RUN_TASK_FP script. &LOAD_MODULES_RUN_TASK_FP; "&RUN_FCST_TN;" "&JOBSDIR;/JREGIONAL_RUN_FCST" {% if machine in ["JET", "HERA"] %} {{ ncores_run_fcst }} - {{ native_run_fcst }} + {{ native_run_fcst }} &RRFS_RESERVE; {% else %} {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} &NCORES_PER_NODE; {% endif %} {{ wtime_run_fcst }} - &RUN_FCST_TN;{{ uscore_ensmem_name }} + &TAG;_&RUN_FCST_TN;{{ uscore_ensmem_name }} &LOGDIR;/&RUN_FCST_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -452,7 +455,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; - &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr# + &TAG;_&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -493,8 +496,8 @@ MODULES_RUN_TASK_FP script. {{ wtime_run_fcst }} 1:ppn=24 - --exclusive - python_@H_skewt + --exclusive &RRFS_RESERVE; + &TAG;_python_@H_skewt &LOGDIR;/python_skewt.log &LOAD_MODULES_RUN_TASK_FP; "run_graphics" "&JOBSDIR;/JREGIONAL_RUN_PYTHON_GRAPHICS" @@ -528,7 +531,7 @@ MODULES_RUN_TASK_FP script. 00:30:00 24G 16 - &RUN_NCL_TN;_#fhr# + &TAG;_&RUN_NCL_TN;_#fhr# &LOGDIR;/&RUN_NCL_TN;_#fhr#_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -560,7 +563,7 @@ MODULES_RUN_TASK_FP script. 1 00:15:00 2G - &RUN_NCL_ZIP_TN;_#fhr# + &TAG;_&RUN_NCL_ZIP_TN;_#fhr# &LOGDIR;/&RUN_NCL_ZIP_TN;_#fhr#_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -590,7 +593,7 @@ MODULES_RUN_TASK_FP script. &JOBSDIR;/../scripts/exregional_clean.ksh 1 00:15:00 - &CLEAN_TN; + &TAG;_&CLEAN_TN; &LOGDIR;/&CLEAN_TN;_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -610,7 +613,7 @@ MODULES_RUN_TASK_FP script. 1 04:00:00 24G - &ARCHIVE_TN; + &TAG;_&ARCHIVE_TN; &LOGDIR;/&ARCHIVE_TN;_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP;