diff --git a/scripts/exregional_clean.ksh b/scripts/exregional_clean.ksh index 6f8c98a08..91891d860 100755 --- a/scripts/exregional_clean.ksh +++ b/scripts/exregional_clean.ksh @@ -78,6 +78,10 @@ for onetime in ${XX[*]};do rm -f ${CYCLE_BASEDIR}/${onetime}/anal_conv_gsi/pe0*_setup rm -f ${CYCLE_BASEDIR}/${onetime}/anal_conv_gsi/obs_input.* rm -f ${CYCLE_BASEDIR}/${onetime}/anal_conv_gsi/diag* + rm -f ${CYCLE_BASEDIR}/${onetime}/anal_conv_gsi_spinup/pe0*.nc4 + rm -f ${CYCLE_BASEDIR}/${onetime}/anal_conv_gsi_spinup/pe0*_setup + rm -f ${CYCLE_BASEDIR}/${onetime}/anal_conv_gsi_spinup/obs_input.* + rm -f ${CYCLE_BASEDIR}/${onetime}/anal_conv_gsi_spinup/diag* rm -rf ${CYCLE_BASEDIR}/${onetime}/fcst_fv3lam/RESTART echo "Deleted netCDF files in ${CYCLE_BASEDIR}/${onetime}/fcst_fv3lam" diff --git a/scripts/exregional_run_analysis.sh b/scripts/exregional_run_analysis.sh index 836db259e..224322a63 100755 --- a/scripts/exregional_run_analysis.sh +++ b/scripts/exregional_run_analysis.sh @@ -84,7 +84,7 @@ case $MACHINE in ulimit -s unlimited ulimit -a export FI_OFI_RXM_SAR_LIMIT=3145728 - export OMP_STACKSIZE=1G + export OMP_STACKSIZE=500M export OMP_NUM_THREADS=${TPP_RUN_ANAL} ncores=$(( NNODES_RUN_ANAL*PPN_RUN_ANAL)) APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_ANAL} --cpu-bind core --depth ${OMP_NUM_THREADS}" @@ -375,7 +375,7 @@ if [ ${regional_ensemble_option:-1} -eq 5 ] && [ ${BKTYPE} != 1 ]; then print_info_msg "$VERBOSE" " Cycle ${YYYYMMDDHH}: GSI hybrid uses FV3LAM ensemble with n_ens=${nummem}" echo " ${YYYYMMDDHH}(${cycle_type}): GSI hybrid uses FV3LAM ensemble with n_ens=${nummem}" >> ${EXPTDIR}/log.cycles grid_ratio_ens="1" - ens_fast_read=.true. + ens_fast_read=.false. else nummem_gfs=$(more filelist03 | wc -l) nummem_gfs=$((nummem_gfs - 3 )) @@ -674,6 +674,7 @@ if [[ ${gsi_type} == "ANALYSIS" && ${anav_type} == "radardbz" ]]; then fi if [[ ${gsi_type} == "ANALYSIS" && ${anav_type} == "conv_dbz" ]]; then ANAVINFO=${FIX_GSI}/${ANAVINFO_CONV_DBZ_FN} + beta1_inv=0.0 if_model_dbz=.true. fi naensloc=`expr ${nsclgrp} \* ${ngvarloc} + ${nsclgrp} - 1` diff --git a/scripts/exregional_run_enkf.sh b/scripts/exregional_run_enkf.sh index f5d900f73..94c310acc 100755 --- a/scripts/exregional_run_enkf.sh +++ b/scripts/exregional_run_enkf.sh @@ -72,10 +72,13 @@ case $MACHINE in ulimit -s unlimited ulimit -a export FI_OFI_RXM_SAR_LIMIT=3145728 - export OMP_STACKSIZE=1G + export OMP_STACKSIZE=2G export OMP_NUM_THREADS=16 + export OMP_PROC_BIND=close + export OMP_PLACES=threads + export MPICH_RANK_REORDER_METHOD=0 ncores=$(( NNODES_RUN_ENKF*PPN_RUN_ENKF )) - APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_ENKF} --cpu-bind core --depth ${OMP_NUM_THREADS}" + APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_ENKF} --label --line-buffer --cpu-bind core --depth ${OMP_NUM_THREADS}" ;; # "THEIA") @@ -489,6 +492,7 @@ fi countdiag=$(ls diag*conv* | wc -l) if [ $countdiag -gt $nens ]; then + if [ ${ob_type} == "conv" ]; then ${APRUN} $enkfworkdir/enkf.x < enkf.nml 1>${stdout_name} 2>${stderr_name} || print_err_msg_exit "\ Call to executable to run EnKF returned with nonzero exit code." @@ -500,6 +504,11 @@ if [ ! -d ${NWGES_DIR}/../enkf_diag ]; then fi cp_vrfy ${stdout_name} ${NWGES_DIR}/../enkf_diag/${stdout_name}.$vlddate cp_vrfy ${stderr_name} ${NWGES_DIR}/../enkf_diag/${stderr_name}.$vlddate + else +${APRUN} $enkfworkdir/enkf.x < enkf.nml 1>${stdout_name} 2>${stderr_name} || print_err_msg_exit "\ +Call to executable to run EnKF returned with nonzero exit code." + echo "Warning: EnKF dbz analysis due to lack of ${ob_type} obs for cycle $vlddate !!!" + fi else echo "Warning: EnKF not running due to lack of ${ob_type} obs for cycle $vlddate !!!" diff --git a/scripts/exregional_run_prdgen.sh b/scripts/exregional_run_prdgen.sh index 760135a54..6db43458a 100755 --- a/scripts/exregional_run_prdgen.sh +++ b/scripts/exregional_run_prdgen.sh @@ -410,6 +410,8 @@ fi fi # block for parallel or series wgrib2 runs. rm_vrfy -rf ${fhr_dir} +rm -fr $DATAprdgen +rm -f $DATA/*.t${cyc}z.*.f${fhr}.*.grib2 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_prepstart.sh b/scripts/exregional_run_prepstart.sh index 6c22e746d..0af8b38f8 100755 --- a/scripts/exregional_run_prepstart.sh +++ b/scripts/exregional_run_prepstart.sh @@ -499,10 +499,10 @@ if [ ${HH} -eq ${SST_update_hour} ] && [ ${cycle_type} == "prod" ] ; then cp ${SST_ROOT}/latest.SST . elif [ -r "${SST_ROOT}/${YYJJJ00000000}" ]; then cp ${SST_ROOT}/${YYJJJ00000000} latest.SST - elif [ -r "${SST_ROOT}/sst.$YYYYMMDD/rtgssthr_grb_0.083.grib2" ]; then - cp ${SST_ROOT}/sst.$YYYYMMDD/rtgssthr_grb_0.083.grib2 latest.SST - elif [ -r "${SST_ROOT}/sst.$YYYYMMDDm1/rtgssthr_grb_0.083.grib2" ]; then - cp ${SST_ROOT}/sst.$YYYYMMDDm1/rtgssthr_grb_0.083.grib2 latest.SST + elif [ -r "${SST_ROOT}/nsst.$YYYYMMDD/rtgssthr_grb_0.083.grib2" ]; then + cp ${SST_ROOT}/nsst.$YYYYMMDD/rtgssthr_grb_0.083.grib2 latest.SST + elif [ -r "${SST_ROOT}/nsst.$YYYYMMDDm1/rtgssthr_grb_0.083.grib2" ]; then + cp ${SST_ROOT}/nsst.$YYYYMMDDm1/rtgssthr_grb_0.083.grib2 latest.SST else echo "${SST_ROOT} data does not exist!!" echo "ERROR: No SST update at ${HH}!!!!" diff --git a/ush/config.sh_rrfs_a_enkf_n3 b/ush/config.sh_rrfs_a_enkf_n3 index 5efdb3cc9..0e3da4b4b 100644 --- a/ush/config.sh_rrfs_a_enkf_n3 +++ b/ush/config.sh_rrfs_a_enkf_n3 @@ -1,11 +1,13 @@ MACHINE="wcoss2" -version="v0.5.3" +MACHINETYPE="backup" +version="v0.6.1" ACCOUNT="RRFS_DEV" #RESERVATION="rrfsdet" EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfs/${version}" EXPT_SUBDIR="rrfs_na_enkf" PREDEF_GRID_NAME="RRFS_NA_3km" + . set_rrfs_config_general.sh DO_ENSEMBLE="TRUE" @@ -28,9 +30,11 @@ DO_SMOKE_DUST="FALSE" #DO_BUFRSND="TRUE" #USE_FVCOM="TRUE" #PREP_FVCOM="TRUE" +DO_PARALLEL_PRDGEN="TRUE" +DO_GSIDIAG_OFFLINE="FALSE" if [[ ${DO_ENSFCST} == "TRUE" ]] ; then - EXPT_SUBDIR="rrfs_conus_enfcst" + EXPT_SUBDIR="rrfs_na_enfcst" DO_SPINUP="FALSE" DO_SAVE_DA_OUTPUT="FALSE" DO_NONVAR_CLDANAL="FALSE" @@ -45,8 +49,8 @@ BOUNDARY_PROC_GROUP_NUM="12" # avaialble retro period: # 20210511-20210531; 20210718-20210801 -DATE_FIRST_CYCL="20230601" -DATE_LAST_CYCL="20230630" +DATE_FIRST_CYCL="20230701" +DATE_LAST_CYCL="20230830" CYCL_HRS=( "00" "12" ) CYCL_HRS=( "18" ) CYCL_HRS_SPINSTART=("06" "18") @@ -55,7 +59,7 @@ if [[ ${DO_ENSFCST} == "TRUE" ]] ; then CYCL_HRS_STOCH=("00" "06" "12" "18") fi #CYCL_HRS_RECENTER=("19") -CYCLEMONTH="06" +CYCLEMONTH="07,08" CYCLEDAY="*" STARTYEAR=${DATE_FIRST_CYCL:0:4} @@ -76,7 +80,7 @@ PRODLONG_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00" #ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}1400 ${DATE_LAST_CYCL}2300 24:00:00" if [[ ${DO_ENSFCST} == "TRUE" ]] ; then BOUNDARY_LEN_HRS="60" - LBC_SPEC_INTVL_HRS="1" + LBC_SPEC_INTVL_HRS="3" DO_SPINUP="FALSE" INITIAL_CYCLEDEF="00 01 01 01 2100 *" BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}1200 ${DATE_LAST_CYCL}2300 06:00:00" @@ -116,7 +120,9 @@ binary_diag=.false. WTIME_RUN_FCST="00:30:00" WTIME_RUN_FCST_LONG="04:45:00" +PPN_RUN_ANAL="16" NNODES_RUN_ANAL="1" +TPP_RUN_ANAL="8" EXTRN_MDL_NAME_ICS="GEFS" EXTRN_MDL_NAME_LBCS="GEFS" @@ -157,16 +163,16 @@ if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then NNODES_RUN_FCST="13" WRTCMP_write_tasks_per_group="7" - NUM_ENS_MEMBERS_FCST=9 + NUM_ENS_MEMBERS_FCST=5 if [[ ${DO_ENSFCST} == "TRUE" ]] ; then NUM_ENS_MEMBERS=${NUM_ENS_MEMBERS_FCST} WTIME_RUN_FCST="04:45:00" WTIME_MAKE_LBCS="01:30:00" - LAYOUT_X="15" - LAYOUT_Y="52" - NNODES_RUN_FCST="13" - WRTCMP_write_tasks_per_group="52" + LAYOUT_X="50" + LAYOUT_Y="64" + NNODES_RUN_FCST="51" + WRTCMP_write_tasks_per_group="64" DO_SPP="TRUE" DO_SPPT="FALSE" @@ -184,21 +190,22 @@ fi RUN_ensctrl="rrfs" RUN="enkfrrfs" -TAG="n3enkf53" +TAG="n3enkf61" if [[ ${DO_ENSFCST} == "TRUE" ]] ; then RUN="refs" - TAG="n3enfcst53" + TAG="n3enfcst61" fi COMINgfs="" . set_rrfs_config.sh STMP="/lfs/f2/t2o/ptmp/emc/stmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. +PTMP="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. +NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory NWGES that save boundary, cold initial, restart files if [[ ${DO_ENSFCST} == "TRUE" ]] ; then STMP="/lfs/f2/t2o/ptmp/emc/stmp/emc.lam/rrfs/${version}/enfcst" # Path to directory STMP that mostly contains input files. + NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges/enfcst" # Path to directory NWGES that save boundary, cold initial, restart files fi -PTMP="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. -NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory NWGES that save boundary, cold initial, restart files ENSCTRL_STMP="/lfs/f2/t2o/ptmp/emc/stmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains control input files for ensemble recentering. ENSCTRL_PTMP="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains control input files for ensemble recentering. ENSCTRL_NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory STMP that mostly contains control input files for ensemble recentering. diff --git a/ush/config.sh_rrfs_a_n3 b/ush/config.sh_rrfs_a_n3 index 44ee14621..5fe9cb638 100644 --- a/ush/config.sh_rrfs_a_n3 +++ b/ush/config.sh_rrfs_a_n3 @@ -1,5 +1,6 @@ MACHINE="wcoss2" -version="v0.5.3" +MACHINETYPE="backup" +version="v0.6.1" ACCOUNT="RRFS_DEV" #RESERVATION="rrfsdet" EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfs/${version}" @@ -41,12 +42,12 @@ BOUNDARY_PROC_GROUP_NUM="72" # avaialble retro period: # 20210511-20210531; 20210718-20210801 -DATE_FIRST_CYCL="20230501" -DATE_LAST_CYCL="20230630" +DATE_FIRST_CYCL="20230701" +DATE_LAST_CYCL="20230830" CYCL_HRS=( "00" "12" ) CYCL_HRS_SPINSTART=("03" "15") CYCL_HRS_PRODSTART=("09" "21") -CYCLEMONTH="05,06" +CYCLEMONTH="07,08" CYCLEDAY="*" STARTYEAR=${DATE_FIRST_CYCL:0:4} @@ -91,7 +92,7 @@ CYCL_HRS_HYB_FV3LAM_ENS=("00" "01" "02" "03" "04" "05" "06" "07" "08" "09" "10" SST_update_hour=01 GVF_update_hour=04 -SNOWICE_update_hour=00 +SNOWICE_update_hour=01 netcdf_diag=.true. binary_diag=.false. @@ -103,7 +104,7 @@ EXTRN_MDL_NAME_LBCS="FV3GFS" envir="para" NET="rrfs_a" -TAG="n3v53" +TAG="n3v61" ARCHIVEDIR="/NCEPDEV/emc-meso/1year/emc.lam/${TAG}" NCL_REGION="conus" @@ -112,11 +113,11 @@ RUN="rrfs" . set_rrfs_config.sh -STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. -PTMP="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. -NWGES="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory NWGES that save boundary, cold initial, restart files +STMP="/lfs/f2/t2o/ptmp/emc/stmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. +PTMP="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. +NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory NWGES that save boundary, cold initial, restart files if [[ ${regional_ensemble_option} == "5" ]]; then - RRFSE_NWGES="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to RRFSE directory NWGES that mostly contains ensemble restart files for GSI hybrid. + RRFSE_NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to RRFSE directory NWGES that mostly contains ensemble restart files for GSI hybrid. NUM_ENS_MEMBERS=30 # FV3LAM ensemble size for GSI hybrid analysis CYCL_HRS_PRODSTART_ENS=( "07" "19" ) DO_ENVAR_RADAR_REF="TRUE" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 074c66bf9..61e32e0c7 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -46,6 +46,9 @@ version="0.1.0" # MACHINE: # Machine on which the workflow will run. # +# MACHINETYPE: +# decide Machine type for wcoss2 (backup or primary) +# # ACCOUNT: # The account under which to submit jobs to the queue. # @@ -145,6 +148,7 @@ version="0.1.0" #----------------------------------------------------------------------- # MACHINE="BIG_COMPUTER" +MACHINETYPE="backup" ACCOUNT="project_name" SERVICE_ACCOUNT="" HPSS_ACCOUNT="" diff --git a/ush/set_rrfs_config.sh b/ush/set_rrfs_config.sh index ce7f9cb06..0bdda4a00 100644 --- a/ush/set_rrfs_config.sh +++ b/ush/set_rrfs_config.sh @@ -37,7 +37,7 @@ if [[ $MACHINE == "wcoss2" ]] ; then SST_ROOT=/lfs/h1/ops/prod/com/gfs/v16.3 GVF_ROOT=/lfs/h1/ops/prod/dcom/viirs IMSSNOW_ROOT=/lfs/h1/ops/prod/com/obsproc/v1.1 - FIRE_RAVE_DIR=/lfs/h2/emc/physics/noscrub/partha.bhattacharjee/RAVE_rawdata/RAVE_NA + FIRE_RAVE_DIR=/lfs/h2/emc/lam/noscrub/emc.lam/RAVE_rawdata/RAVE_NA FVCOM_DIR="/lfs/h1/ops/prod/com/nosofs/v3.5" FVCOM_FILE="fvcom" fi diff --git a/ush/set_rrfs_config_general.sh b/ush/set_rrfs_config_general.sh index 18f7a59a4..86ac0335a 100644 --- a/ush/set_rrfs_config_general.sh +++ b/ush/set_rrfs_config_general.sh @@ -56,6 +56,13 @@ if [[ $MACHINE == "wcoss2" ]] ; then QUEUE_HPSS="dev_transfer" QUEUE_PRDGEN="pridev" QUEUE_GRAPHICS="pridev" + if [[ $MACHINETYPE == "backup" ]] ; then + QUEUE_DEFAULT="devhigh" + QUEUE_FCST="devmax" + QUEUE_ANALYSIS="devmax" + QUEUE_PRDGEN="devhigh" + QUEUE_POST="devhigh" + fi fi VERBOSE="TRUE" @@ -195,6 +202,7 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_3km" ]] ; then MEMO_RUN_PROCESSBUFR="80G" PPN_RUN_NONVARCLDANL="64" NNODES_RUN_NONVARCLDANL="10" + PPN_RUN_GSIDIAG=128 MAXTRIES_GET_EXTRN_ICS="1" MAXTRIES_GET_EXTRN_LBCS="1" @@ -277,6 +285,7 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_13km" ]] ; then NNODES_RUN_ENKF="2" PPN_RUN_NONVARCLDANL="128" NNODES_RUN_NONVARCLDANL="1" + PPN_RUN_GSIDIAG=128 WTIME_GET_EXTRN_ICS="00:05:00" WTIME_GET_EXTRN_LBCS="00:05:00" @@ -398,9 +407,9 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_NA_3km" ]] ; then TPP_MAKE_LBCS="2" NNODES_MAKE_ICS="2" NNODES_MAKE_LBCS="1" - PPN_RUN_ANAL="16" - NNODES_RUN_ANAL="22" - TPP_RUN_ANAL="8" + PPN_RUN_ANAL="8" + NNODES_RUN_ANAL="60" + TPP_RUN_ANAL="16" PPN_RUN_FCST="64" NNODES_RUN_FCST="51" TPP_RUN_FCST="2" @@ -412,10 +421,11 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_NA_3km" ]] ; then PPN_RUN_ENKF="8" NNODES_RUN_ENKF="10" TPP_RUN_ENKF="16" - PPN_RUN_NONVARCLDANL="64" - NNODES_RUN_NONVARCLDANL="5" + PPN_RUN_NONVARCLDANL="128" + NNODES_RUN_NONVARCLDANL="1" PPN_RUN_PRDGEN="128" PPN_RUN_POSTANAL="8" + PPN_RUN_GSIDIAG=128 MAXTRIES_MAKE_ICS="1" MAXTRIES_MAKE_LBCS="1" @@ -445,13 +455,13 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_NA_3km" ]] ; then FV3GFS_FILE_FMT_LBCS="netcdf" WTIME_RUN_FCST="02:30:00" - WTIME_RUN_FCST_LONG="04:30:00" + WTIME_RUN_FCST_LONG="05:30:00" WTIME_MAKE_LBCS="02:30:00" - CLEAN_OLDPROD_HRS="72" - CLEAN_OLDLOG_HRS="72" + CLEAN_OLDPROD_HRS="120" + CLEAN_OLDLOG_HRS="120" CLEAN_OLDRUN_HRS="18" - CLEAN_OLDFCST_HRS="12" + CLEAN_OLDFCST_HRS="18" CLEAN_OLDSTMPPOST_HRS="12" CLEAN_NWGES_HRS="18" diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index ba482d0fd..34df7a145 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -359,7 +359,7 @@ define resources used for each tasks {%- if do_retro %} {% else %} - + {%- endif %} {# Double quotes are required inside the strftime! Expect an error from reading the template if using single quotes. #} {{ at_start_cycledef }} @@ -749,7 +749,7 @@ MODULES_RUN_TASK_FP script. -{%- if extrn_mdl_name_lbcs in ["GEFS"] and bc_update_interval==1 %} +{%- if extrn_mdl_name_lbcs in ["GEFS"] %}