diff --git a/env/HERA.env b/env/HERA.env index f6c7c6543d1..b4d722b590e 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -214,11 +214,9 @@ elif [ $step = "epos" ]; then [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max export APRUN_EPOS="$launcher" -elif [ $step = "fv3ic" ]; then +elif [ $step = "init" ]; then - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" + export APRUN="$launcher" elif [ $step = "postsnd" ]; then diff --git a/env/JET.env b/env/JET.env index d4d0cdb2a53..8bab0b7ef21 100755 --- a/env/JET.env +++ b/env/JET.env @@ -179,11 +179,9 @@ elif [ $step = "epos" ]; then [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max export APRUN_EPOS="$launcher ${npe_epos:-$PBS_NP}" -elif [ $step = "fv3ic" ]; then +elif [ $step = "init" ]; then - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" + export APRUN="$launcher" elif [ $step = "postsnd" ]; then diff --git a/env/ORION.env b/env/ORION.env index de989331ff1..69721c21edc 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -202,11 +202,9 @@ elif [ $step = "epos" ]; then [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max export APRUN_EPOS="$launcher" -elif [ $step = "fv3ic" ]; then +elif [ $step = "init" ]; then - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" + export APRUN="$launcher" elif [ $step = "postsnd" ]; then diff --git a/env/WCOSS_C.env b/env/WCOSS_C.env index 17396abce7a..f79586a8840 100755 --- a/env/WCOSS_C.env +++ b/env/WCOSS_C.env @@ -198,11 +198,9 @@ elif [ $step = "epos" ]; then [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max export APRUN_EPOS="$launcher -j 1 -n $npe_epos -N $npe_node_epos -d $NTHREADS_EPOS -cc depth" -elif [ $step = "fv3ic" ]; then +elif [ $step = "init" ]; then - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="$launcher -j 1 -n 1 -N 1 -d $NTHREADS_CHGRES -cc depth" + export APRUN="$launcher" elif [ $step = "vrfy" ]; then diff --git a/env/WCOSS_DELL_P3.env b/env/WCOSS_DELL_P3.env index 46c50b07666..e408c4850bd 100755 --- a/env/WCOSS_DELL_P3.env +++ b/env/WCOSS_DELL_P3.env @@ -204,11 +204,9 @@ elif [ $step = "epos" ]; then [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max export APRUN_EPOS="$launcher ${npe_epos:-$PBS_NP}" -elif [ $step = "fv3ic" ]; then +elif [ $step = "init" ]; then - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" + export APRUN="mpirun" elif [ $step = "postsnd" ]; then diff --git a/jobs/JGLOBAL_WAVE_PREP b/jobs/JGLOBAL_WAVE_PREP index 4fe921c9aba..138a3bf64d8 100755 --- a/jobs/JGLOBAL_WAVE_PREP +++ b/jobs/JGLOBAL_WAVE_PREP @@ -77,6 +77,10 @@ else if [ ! -L $ROTDIR/${WAVECUR_DID}.${RPDY} ]; then # Check if symlink already exists in ROTDIR $NLN $DMPDIR/${WAVECUR_DID}.${RPDY} $ROTDIR/${WAVECUR_DID}.${RPDY} fi + BRPDY=`$NDATE -24 ${RPDY}00 | cut -c1-8` + if [ ! -L $ROTDIR/${WAVECUR_DID}.${BRPDY} ]; then # Check if symlink already exists in ROTDIR + $NLN $DMPDIR/${WAVECUR_DID}.${BRPDY} $ROTDIR/${WAVECUR_DID}.${BRPDY} + fi if [ ! -L $ROTDIR/${CDUMP}.${PDY}/${cyc}/atmos/${WAVICEFILE} ]; then # Check if symlink already exists in ROTDIR $NLN $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} $ROTDIR/$CDUMP.${PDY}/$cyc/atmos/${WAVICEFILE} fi diff --git a/jobs/rocoto/arch.sh b/jobs/rocoto/arch.sh index b33857838f2..8f77ee2d693 100755 --- a/jobs/rocoto/arch.sh +++ b/jobs/rocoto/arch.sh @@ -182,7 +182,13 @@ if [ $CDUMP = "gfs" ]; then #for targrp in gfs_flux gfs_netcdf/nemsio gfs_pgrb2b; do if [ ${SAVEFCSTNEMSIO:-"YES"} = "YES" ]; then - for targrp in gfs_flux gfs_${format}a gfs_${format}b gfs_pgrb2b; do + if [ $MODE = "cycled" ]; then + targrp_list=`gfs_flux gfs_${format}a gfs_${format}b gfs_pgrb2b` + elif [ $MODE = "free" ]; then + targrp_list=`gfs_flux gfs_${format}b gfs_pgrb2b` + fi + #for targrp in gfs_flux gfs_${format}a gfs_${format}b gfs_pgrb2b; do + for targrp in $targrp_list; do htar -P -cvf $ATARDIR/$CDATE/${targrp}.tar `cat $ARCH_LIST/${targrp}.txt` status=$? if [ $status -ne 0 -a $CDATE -ge $firstday ]; then diff --git a/jobs/rocoto/fv3ic.sh b/jobs/rocoto/fv3ic.sh deleted file mode 100755 index b5c2e40b484..00000000000 --- a/jobs/rocoto/fv3ic.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/ksh -x - -############################################################### -## Abstract: -## Create FV3 initial conditions from GFS intitial conditions -## RUN_ENVIR : runtime environment (emc | nco) -## HOMEgfs : /full/path/to/workflow -## EXPDIR : /full/path/to/config/files -## CDATE : current date (YYYYMMDDHH) -## CDUMP : cycle name (gdas / gfs) -## PDY : current date (YYYYMMDD) -## cyc : current cycle (HH) -############################################################### - -############################################################### -# Source FV3GFS workflow modules -. $HOMEgfs/ush/load_fv3gfs_modules.sh -status=$? -[[ $status -ne 0 ]] && exit $status - -############################################################### -# Source relevant configs -configs="base fv3ic" -for config in $configs; do - . $EXPDIR/config.${config} - status=$? - [[ $status -ne 0 ]] && exit $status -done - -############################################################### -# Source machine runtime environment -. $BASE_ENV/${machine}.env fv3ic -status=$? -[[ $status -ne 0 ]] && exit $status - -# Set component -export COMPONENT=${COMPONENT:-atmos} - -# Temporary runtime directory -export DATA="$RUNDIR/$CDATE/$CDUMP/fv3ic$$" -[[ -d $DATA ]] && rm -rf $DATA - -# Input GFS initial condition directory -export INIDIR="$ICSDIR/$CDATE/$CDUMP/$CDUMP.$PDY/$cyc" - -# Output FV3 initial condition directory -export OUTDIR="$ICSDIR/$CDATE/$CDUMP/$CASE/INPUT" - -export OMP_NUM_THREADS_CH=$NTHREADS_CHGRES -export APRUNC=$APRUN_CHGRES - -# Call global_chgres_driver.sh -$HOMEgfs/ush/global_chgres_driver.sh -status=$? -if [ $status -ne 0 ]; then - echo "global_chgres_driver.sh returned with a non-zero exit code, ABORT!" - exit $status -fi - -# Stage the FV3 initial conditions to ROTDIR -COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT" -[[ ! -d $COMOUT ]] && mkdir -p $COMOUT -cd $COMOUT || exit 99 -rm -rf INPUT -$NLN $OUTDIR . - -############################################################### -# Exit cleanly -exit 0 diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index 120e3b9454f..f08a10a8dd4 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -20,7 +20,7 @@ status=$? ############################################################### # Source relevant configs -configs="base getic" +configs="base getic init" for config in $configs; do . $EXPDIR/config.${config} status=$? @@ -36,168 +36,125 @@ status=$? ############################################################### # Set script and dependency variables -yyyy=$(echo $CDATE | cut -c1-4) -mm=$(echo $CDATE | cut -c5-6) -dd=$(echo $CDATE | cut -c7-8) -cyc=${cyc:-$(echo $CDATE | cut -c9-10)} - -export COMPONENT=${COMPONENT:-atmos} +export yy=$(echo $CDATE | cut -c1-4) +export mm=$(echo $CDATE | cut -c5-6) +export dd=$(echo $CDATE | cut -c7-8) +export hh=${cyc:-$(echo $CDATE | cut -c9-10)} +export GDATE=$($NDATE -${assim_freq:-"06"} $CDATE) +export gyy=$(echo $GDATE | cut -c1-4) +export gmm=$(echo $GDATE | cut -c5-6) +export gdd=$(echo $GDATE | cut -c7-8) +export ghh=$(echo $GDATE | cut -c9-10) + +export DATA=${DATA:-${DATAROOT}/getic} +export EXTRACT_DIR=${DATA:-$EXTRACT_DIR} +export PRODHPSSDIR=${PRODHPSSDIR:-/NCEPPROD/hpssprod/runhistory} +export COMPONENT="atmos" +export gfs_ver=${gfs_ver:-"v16"} +export OPS_RES=${OPS_RES:-"C768"} +export GETICSH=${GETICSH:-${GDASINIT_DIR}/get_v16.data.sh} + +# Create ROTDIR/EXTRACT_DIR +if [ ! -d $ROTDIR ]; then mkdir -p $ROTDIR ; fi +if [ ! -d $EXTRACT_DIR ]; then mkdir -p $EXTRACT_DIR ; fi +cd $EXTRACT_DIR + +# Check version, cold/warm start, and resolution +if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; then # Pull warm start ICs - no chgres + + # Pull RESTART files off HPSS + if [ ${RETRO:-"NO"} = "YES" ]; then # Retrospective parallel input + + # Pull prior cycle restart files + htar -xvf ${HPSSDIR}/${GDATE}/gdas_restartb.tar + status=$? + [[ $status -ne 0 ]] && exit $status -############################################################### + # Pull current cycle restart files + htar -xvf ${HPSSDIR}/${CDATE}/gfs_restarta.tar + status=$? + [[ $status -ne 0 ]] && exit $status -target_dir=$ICSDIR/$CDATE/$CDUMP -mkdir -p $target_dir -cd $target_dir - -# Initialize return code to 0 -rc=1 - -if [ $ics_from = "opsgfs" ]; then - - # Location of production tarballs on HPSS - hpssdir="/NCEPPROD/hpssprod/runhistory/rh$yyyy/$yyyy$mm/$PDY" - - # Handle nemsio and pre-nemsio GFS filenames - if [ $CDATE -le "2019061118" ]; then #GFSv14 - # Add CDUMP.PDY/CYC to target_dir - target_dir=$ICSDIR/$CDATE/$CDUMP/${CDUMP}.$yyyy$mm$dd/$cyc - mkdir -p $target_dir - cd $target_dir - - nfanal=4 - fanal[1]="./${CDUMP}.t${cyc}z.atmanl.nemsio" - fanal[2]="./${CDUMP}.t${cyc}z.sfcanl.nemsio" - fanal[3]="./${CDUMP}.t${cyc}z.nstanl.nemsio" - fanal[4]="./${CDUMP}.t${cyc}z.pgrbanl" - flanal="${fanal[1]} ${fanal[2]} ${fanal[3]} ${fanal[4]}" - tarpref="gpfs_hps_nco_ops_com" - if [ $CDUMP = "gdas" ]; then - tarball="$hpssdir/${tarpref}_gfs_prod_${CDUMP}.${CDATE}.tar" - elif [ $CDUMP = "gfs" ]; then - tarball="$hpssdir/${tarpref}_gfs_prod_${CDUMP}.${CDATE}.anl.tar" - fi - else #GFSv15 - nfanal=2 - fanal[1]="./${CDUMP}.$yyyy$mm$dd/$cyc/${CDUMP}.t${cyc}z.atmanl.nemsio" - fanal[2]="./${CDUMP}.$yyyy$mm$dd/$cyc/${CDUMP}.t${cyc}z.sfcanl.nemsio" - flanal="${fanal[1]} ${fanal[2]}" - if [ $CDATE -ge "2020022600" ]; then - tarpref="com" - else - tarpref="gpfs_dell1_nco_ops_com" - fi - if [ $CDUMP = "gdas" ]; then - tarball="$hpssdir/${tarpref}_gfs_prod_${CDUMP}.${yyyy}${mm}${dd}_${cyc}.${CDUMP}_nemsio.tar" - elif [ $CDUMP = "gfs" ]; then - tarball="$hpssdir/${tarpref}_gfs_prod_${CDUMP}.${yyyy}${mm}${dd}_${cyc}.${CDUMP}_nemsioa.tar" - fi - fi - - # First check the COMROOT for files, if present copy over - if [ $machine = "WCOSS_C" ]; then - - # Need COMROOT - module load prod_envir/1.1.0 >> /dev/null 2>&1 - - comdir="$COMROOT/$CDUMP/prod/$CDUMP.$PDY" - rc=0 - for i in `seq 1 $nfanal`; do - if [ -f $comdir/${fanal[i]} ]; then - $NCP $comdir/${fanal[i]} ${fanal[i]} - else - rb=1 ; ((rc+=rb)) - fi - done - - fi - - # Get initial conditions from HPSS - if [ $rc -ne 0 ]; then - - # check if the tarball exists - hsi ls -l $tarball - rc=$? - if [ $rc -ne 0 ]; then - echo "$tarball does not exist and should, ABORT!" - exit $rc - fi - # get the tarball - htar -xvf $tarball $flanal - rc=$? - if [ $rc -ne 0 ]; then - echo "untarring $tarball failed, ABORT!" - exit $rc - fi + # Pull IAU increment files + htar -xvf ${HPSSDIR}/${CDATE}/gfs_netcdfa.tar + status=$? + [[ $status -ne 0 ]] && exit $status - # Move the files to legacy EMC filenames - if [ $CDATE -le "2019061118" ]; then #GFSv14 - for i in `seq 1 $nfanal`; do - $NMV ${fanal[i]} ${flanal[i]} - done - fi + else # Opertional input - warm starts - fi - - # If found, exit out - if [ $rc -ne 0 ]; then - echo "Unable to obtain operational GFS initial conditions, ABORT!" - exit 1 - fi - -elif [ $ics_from = "pargfs" ]; then - - # Add CDUMP.PDY/CYC to target_dir - target_dir=$ICSDIR/$CDATE/$CDUMP/${CDUMP}.$yyyy$mm$dd/$cyc - mkdir -p $target_dir - cd $target_dir - - # Filenames in parallel - nfanal=4 - fanal[1]="gfnanl.${CDUMP}.$CDATE" - fanal[2]="sfnanl.${CDUMP}.$CDATE" - fanal[3]="nsnanl.${CDUMP}.$CDATE" - fanal[4]="pgbanl.${CDUMP}.$CDATE" - flanal="${fanal[1]} ${fanal[2]} ${fanal[3]} ${fanal[4]}" - - # Get initial conditions from HPSS from retrospective parallel - tarball="$HPSS_PAR_PATH/${CDATE}${CDUMP}.tar" - - # check if the tarball exists - hsi ls -l $tarball - rc=$? - if [ $rc -ne 0 ]; then - echo "$tarball does not exist and should, ABORT!" - exit $rc - fi - # get the tarball - htar -xvf $tarball $flanal - rc=$? - if [ $rc -ne 0 ]; then - echo "untarring $tarball failed, ABORT!" - exit $rc - fi - - # If found, exit out - if [ $rc -ne 0 ]; then - echo "Unable to obtain parallel GFS initial conditions, ABORT!" - exit 1 - fi + # Pull CDATE gfs restart tarball + htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_restart.tar + # Pull GDATE gdas restart tarball + htar -xvf ${PRODHPSSDIR}/rh${gyy}/${gyy}${gmm}/${gyy}${gmm}${gdd}/com_gfs_prod_gdas.${gyy}${gmm}${gdd}_${ghh}.gdas_restart.tar + fi -else +else # Pull chgres cube inputs for cold start IC generation - echo "ics_from = $ics_from is not supported, ABORT!" - exit 1 + # Run UFS_UTILS GETICSH + sh ${GETICSH} ${CDUMP} + status=$? + [[ $status -ne 0 ]] && exit $status fi -############################################################### -# Copy pgbanl file to COMROT for verification - GFSv14 only -if [ $CDATE -le "2019061118" ]; then #GFSv14 - COMROT=$ROTDIR/${CDUMP}.$PDY/$cyc/$COMPONENT - [[ ! -d $COMROT ]] && mkdir -p $COMROT - $NCP ${fanal[4]} $COMROT/${CDUMP}.t${cyc}z.pgrbanl +# Move extracted data to ROTDIR +if [ ! -d ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT} ]; then mkdir -p ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT} ; fi +if [ $gfs_ver = v16 -a $RETRO = "YES" ]; then + mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/* ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT} +else + mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/* ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh} fi +# Pull pgbanl file for verification/archival - v14+ +if [ $gfs_ver = v14 -o $gfs_ver = v15 -o $gfs_ver = v16 ]; then + for grid in 0p25 0p50 1p00 + do + file=gfs.t${hh}z.pgrb2.${grid}.anl + + if [ $gfs_ver = v14 ]; then # v14 production source + + cd $ROTDIR/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT} + export tarball="gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}${hh}.pgrb2_${grid}.tar" + htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${file} + + elif [ $gfs_ver = v15 ]; then # v15 production source + + cd $EXTRACT_DIR + export tarball="com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar" + htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file} + mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file} + + elif [ $gfs_ver = v16 ]; then # v16 - determine RETRO or production source next + + if [ $RETRO = "YES" ]; then # Retrospective parallel source + + cd $EXTRACT_DIR + if [ $grid = "0p25" ]; then # anl file spread across multiple tarballs + export tarball="gfsa.tar" + elif [ $grid = "0p50" -o $grid = "1p00" ]; then + export tarball="gfsb.tar" + fi + htar -xvf ${HPSSDIR}/${yy}${mm}${dd}${hh}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file} + mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file} + + else # Production source + + cd $ROTDIR + export tarball="com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar" + htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/atmos/${file} + + fi # RETRO vs production + + fi # Version check + done # grid loop +fi # v14-v16 pgrb anl file pull + +########################################## +# Remove the Temporary working directory +########################################## +cd $DATAROOT +[[ $KEEPDATA = "NO" ]] && rm -rf $DATA + ############################################################### # Exit out cleanly exit 0 diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh new file mode 100755 index 00000000000..d76233f92f5 --- /dev/null +++ b/jobs/rocoto/init.sh @@ -0,0 +1,73 @@ +#!/bin/ksh -x + +############################################################### +## Abstract: +## Get GFS intitial conditions +## RUN_ENVIR : runtime environment (emc | nco) +## HOMEgfs : /full/path/to/workflow +## EXPDIR : /full/path/to/config/files +## CDATE : current date (YYYYMMDDHH) +## CDUMP : cycle name (gdas / gfs) +## PDY : current date (YYYYMMDD) +## cyc : current cycle (HH) +############################################################### + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Source relevant configs +configs="base getic init" +for config in $configs; do + . $EXPDIR/config.${config} + status=$? + [[ $status -ne 0 ]] && exit $status +done + +############################################################### +# Source machine runtime environment +. $BASE_ENV/${machine}.env init +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Set script and dependency variables + +export yy=$(echo $CDATE | cut -c1-4) +export mm=$(echo $CDATE | cut -c5-6) +export dd=$(echo $CDATE | cut -c7-8) +export hh=${cyc:-$(echo $CDATE | cut -c9-10)} + +export DATA=${DATA:-${DATAROOT}/init} +export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR} +export WORKDIR=${WORKDIR:-$DATA} +export OUTDIR=${OUTDIR:-$ROTDIR} +export COMPONENT="atmos" +export gfs_ver=${gfs_ver:-"v16"} +export OPS_RES=${OPS_RES:-"C768"} +export RUNICSH=${RUNICSH:-${GDASINIT_DIR}/run_v16.chgres.sh} + +# Check if init is needed and run if so +if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; then + echo "Detected v16 $OPS_RES warm starts, will not run init. Exiting..." + exit 0 +else + # Run chgres_cube + if [ ! -d $OUTDIR ]; then mkdir -p $OUTDIR ; fi + sh ${RUNICSH} ${CDUMP} + status=$? + [[ $status -ne 0 ]] && exit $status +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd $DATAROOT +[[ $KEEPDATA = "NO" ]] && rm -rf $DATA + +############################################################### +# Exit out cleanly +exit 0 diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index 1a164b1e867..e68d0c0b0ab 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -58,6 +58,8 @@ export WAFSF="NO" # WAFS products # use RUNMOS flag (currently in config.vrfy) export REALTIME="YES" +# Experiment mode (cycled or free-forecast) +export MODE="@MODE@" # cycled/free #################################################### # DO NOT ADD MACHINE DEPENDENT STUFF BELOW THIS LINE @@ -138,6 +140,7 @@ export DBNROOT=${DBNROOT:-${UTILROOT}/fakedbn} export LEVS=128 export CASE="@CASECTL@" export CASE_ENKF="@CASEENS@" +export OPS_RES="C768" # Run with CCPP physics export RUN_CCPP="YES" @@ -209,7 +212,8 @@ export IAU_OFFSET=6 export DOIAU_ENKF=${DOIAU:-"YES"} # Enable 4DIAU for EnKF ensemble export IAUFHRS_ENKF="3,6,9" export IAU_DELTHRS_ENKF=6 -if [[ "$SDATE" = "$CDATE" && $EXP_WARM_START = ".false." ]] || [[ "$DOIAU" = "NO" ]] ; then # Cold starting or IAU off +# Check if cycle is cold starting, DOIAU off, or free-forecast mode +if [[ "$MODE" = "cycled" && "$SDATE" = "$CDATE" && $EXP_WARM_START = ".false." ]] || [[ "$DOIAU" = "NO" ]] || [[ "$MODE" = "free" && $EXP_WARM_START = ".false." ]] ; then export IAU_OFFSET=0 export IAU_FHROT=0 fi @@ -223,6 +227,7 @@ export gldas_cyc=00 # run wave component export DO_WAVE="NO" +if [[ "$SDATE" -lt "2019020100" ]]; then DO_WAVE="NO" ; fi # no rtofs in GDA export WAVE_CDUMP="both" # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 index e70304bfa9f..1c4025f84c6 100755 --- a/parm/config/config.fv3 +++ b/parm/config/config.fv3 @@ -36,13 +36,13 @@ fi # (Standard) Model resolution dependent variables case $case_in in "C48") - export DELTIM=450 - export layout_x=2 - export layout_y=4 - export layout_x_gfs=2 - export layout_y_gfs=4 - export npe_wav=140 - export npe_wav_gfs=140 + export DELTIM=1200 + export layout_x=3 + export layout_y=2 + export layout_x_gfs=3 + export layout_y_gfs=2 + export npe_wav=16 + export npe_wav_gfs=16 export nth_fv3=1 export nth_fv3_gfs=1 export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling @@ -50,16 +50,16 @@ case $case_in in export WRTTASK_PER_GROUP=$npe_node_max export WRITE_GROUP_GFS=1 export WRTTASK_PER_GROUP_GFS=$npe_node_max - export WRTIOBUF="4M" + export WRTIOBUF="1M" ;; "C96") - export DELTIM=450 - export layout_x=4 + export DELTIM=720 + export layout_x=6 export layout_y=4 - export layout_x_gfs=4 + export layout_x_gfs=6 export layout_y_gfs=4 - export npe_wav=140 - export npe_wav_gfs=140 + export npe_wav=40 + export npe_wav_gfs=40 export nth_fv3=1 export nth_fv3_gfs=1 export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling @@ -68,6 +68,7 @@ case $case_in in export WRITE_GROUP_GFS=1 export WRTTASK_PER_GROUP_GFS=$npe_node_max export WRTIOBUF="4M" + export n_split=6 ;; "C192") export DELTIM=450 @@ -88,14 +89,14 @@ case $case_in in ;; "C384") export DELTIM=240 - export layout_x=8 + export layout_x=6 export layout_y=8 export layout_x_gfs=6 - export layout_y_gfs=6 + export layout_y_gfs=8 export npe_wav=140 export npe_wav_gfs=140 - export nth_fv3=1 - export nth_fv3_gfs=1 + export nth_fv3=2 + export nth_fv3_gfs=2 export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 export WRTTASK_PER_GROUP=$npe_node_max diff --git a/parm/config/config.getic b/parm/config/config.getic index a7544542641..ced2c051f00 100755 --- a/parm/config/config.getic +++ b/parm/config/config.getic @@ -8,13 +8,54 @@ echo "BEGIN: config.getic" # Get task specific resources . $EXPDIR/config.resources getic -# We should just be supporting the OPSGFS only -export ics_from="opsgfs" # initial conditions from opsgfs or pargfs +export RETRO="NO" # YES = Pull v16 inputs from retrospective parallels; NO = use operational inputs +export gfs_ver="v16" # Default = v16 -# Provide a parallel experiment name and path to HPSS archive -if [ $ics_from = "pargfs" ]; then - export parexp="prnemsrn" - export HPSS_PAR_PATH="/5year/NCEPDEV/emc-global/emc.glopara/WCOSS_C/$parexp" +export UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd +export GDASINIT_DIR=${UFS_DIR}/util/gdas_init + +export PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory +export GETICSH=${GDASINIT_DIR}/get_v16.data.sh + +if [ ${RETRO:-"NO"} = "YES" ]; then # Retrospective parallel input + export GETICSH=${GDASINIT_DIR}/get_v16retro.data.sh + if [[ "$CDATE" -lt "2019060106" ]]; then + HPSSDIR=/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_D/gfsv16/v16retro0e + elif [[ "$CDATE" -lt "2019090100" ]]; then + HPSSDIR=/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_D/gfsv16/v16retro1e + elif [[ "$CDATE" -lt "2019101706" ]]; then + HPSSDIR=/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_D/gfsv16/v16retro2e + elif [[ "$CDATE" -lt "2020122206" ]]; then + HPSSDIR=/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_D/gfsv16/v16rt2 + else + HPSSDIR=/NCEPDEV/emc-global/5year/emc.gloparadev/WCOSS_D/gfsv16/v16rt2n + fi +elif [ ${RETRO:-"NO"} = "NO" ]; then # Operational input + # No ENKF data prior to 2012/05/21/00z + if [[ "$CDATE" -lt "2012052100" ]]; then + set +x + echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA + elif [[ "$CDATE" -lt "2016051000" ]]; then + export gfs_ver=v12 + export GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh + elif [[ "$CDATE" -lt "2017072000" ]]; then + export gfs_ver=v13 + export GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh + elif [[ "$CDATE" -lt "2019061200" ]]; then + export gfs_ver=v14 + export GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh + elif [[ "$CDATE" -lt "2021032100" ]]; then + export gfs_ver=v15 + export GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh + elif [[ "$CDATE" -lt "2021032106" ]]; then + # The way the v16 switch over was done, there is no complete + # set of v16 or v15 data for 2021032100. And although + # v16 was officially implemented 2021032212, the v16 prod + # tarballs were archived starting 2021032106. + set +x + echo FATAL ERROR: NO V15 OR V16 DATA FOR 2021032100 + exit 1 + fi fi echo "END: config.getic" diff --git a/parm/config/config.init b/parm/config/config.init new file mode 100755 index 00000000000..eeb04a7d409 --- /dev/null +++ b/parm/config/config.init @@ -0,0 +1,53 @@ +#!/bin/ksh -x + +########## config.init ########## +# Prepare initial conditions + +echo "BEGIN: config.init" + +# Get task specific resources +. $EXPDIR/config.resources init + +# Get task specific resources +. $EXPDIR/config.getic + +export UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd +export GDASINIT_DIR=${UFS_DIR}/util/gdas_init + +export CRES_HIRES=$CASE +export CRES_ENKF=$CASE_ENKF + +export RUNICSH=${GDASINIT_DIR}/run_v16.chgres.sh +if [ "${RETRO:-"NO"}" = "YES" ] || [ "$CDUMP" = "gdas" ]; then + export RUNICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh +fi + +if [ ${RETRO:-"NO"} = "NO" ]; then # Operational input + # No ENKF data prior to 2012/05/21/00z + if [[ "$CDATE" -lt "2012052100" ]]; then + set +x + echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA + elif [[ "$CDATE" -lt "2016051000" ]]; then + export gfs_ver=v12 + export RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh + elif [[ "$CDATE" -lt "2017072000" ]]; then + export gfs_ver=v13 + export RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh + elif [[ "$CDATE" -lt "2019061200" ]]; then + export gfs_ver=v14 + export RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh + elif [[ "$CDATE" -lt "2021032100" ]]; then + export gfs_ver=v15 + export RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh + elif [[ "$CDATE" -lt "2021032106" ]]; then + # The way the v16 switch over was done, there is no complete + # set of v16 or v15 data for 2021032100. And although + # v16 was officially implemented 2021032212, the v16 prod + # tarballs were archived starting 2021032106. + set +x + echo FATAL ERROR: NO V15 OR V16 DATA FOR 2021032100 + exit 1 + fi +fi + +echo "END: config.init" diff --git a/parm/config/config.nsst b/parm/config/config.nsst index a59ca19495f..ef8767834d1 100755 --- a/parm/config/config.nsst +++ b/parm/config/config.nsst @@ -12,9 +12,9 @@ export NST_MODEL=2 # nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON, export NST_SPINUP=0 -#if [[ "$CDATE" = $SDATE ]]; then -# export NST_SPINUP=1 -#fi +if [[ "$CDATE" -lt "2017072000" ]]; then + export NST_SPINUP=1 +fi # nstf_name(3) : NST_RESV (Reserved, NSST Analysis) : 0 = OFF, 1 = ON export NST_RESV=0 diff --git a/parm/config/config.resources b/parm/config/config.resources index 7d284f8b7f4..f404ad8b1b3 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -8,6 +8,7 @@ if [ $# -ne 1 ]; then echo "Must specify an input task argument to set resource variables!" echo "argument can be any one of the following:" + echo "getic init" echo "anal analcalc analdiag gldas fcst post vrfy metp arch echgres" echo "eobs ediag eomg eupd ecen esfc efcs epos earc" echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt" @@ -270,6 +271,13 @@ elif [ $step = "echgres" ]; then export nth_echgres=$npe_node_max export npe_node_echgres=1 +elif [ $step = "init" ]; then + + export wtime_init="00:30:00" + export npe_init=24 + export nth_init=1 + export npe_node_init=6 + elif [ $step = "arch" -o $step = "earc" -o $step = "getic" ]; then eval "export wtime_$step='06:00:00'" diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 45f74796dd1..4ee72c3c2d7 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -624,7 +624,7 @@ if [ ${CFP_MP:-"NO"} = "YES" ]; then nm=0; fi - if [ ${CFP_MP:-"NO"} = "YES" ]; then + if [ ${CFP_MP:-"NO"} = "YES" ] && [ "$DOBLL_WAV" = "YES" ]; then if [ "$DOBNDPNT_WAV" = YES ]; then if [ "$DOSPC_WAV" = YES ]; then echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile diff --git a/sorc/build_all.sh b/sorc/build_all.sh index acba8d6a2cc..6c027efc792 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -9,16 +9,6 @@ set +x # Anything other than "true" will use libraries locally. #------------------------------------ -while getopts "c" option; -do - case $option in - c) - echo "Received -c flag, check out ufs-weather-model develop branch with CCPP physics" - RUN_CCPP="YES" - ;; - esac -done - export USE_PREINST_LIBS="true" #------------------------------------ @@ -61,7 +51,6 @@ err=0 #------------------------------------ $Build_fv3gfs && { echo " .... Building fv3 .... " -export RUN_CCPP=${RUN_CCPP:-"NO"} ./build_fv3.sh > $logs_dir/build_fv3.log 2>&1 rc=$? if [[ $rc -ne 0 ]] ; then diff --git a/sorc/build_fv3.sh b/sorc/build_fv3.sh index 2c3efdadec3..67c28e02ae8 100755 --- a/sorc/build_fv3.sh +++ b/sorc/build_fv3.sh @@ -23,10 +23,6 @@ cd fv3gfs.fd/ FV3=$( pwd -P )/FV3 cd tests/ if [ ! -d ../NEMS/exe ]; then mkdir ../NEMS/exe ; fi -if [ ${RUN_CCPP:-${1:-"YES"}} = "NO" ]; then - ./compile.sh "$FV3" "$target" "WW3=Y 32BIT=Y" 1 - mv -f fv3_1.exe ../NEMS/exe/global_fv3gfs.x -else - ./compile.sh "$target" "APP=ATM CCPP=Y 32BIT=Y SUITES=FV3_GFS_v16,FV3_GFS_v16_RRTMGP" 2 YES NO - mv -f fv3_2.exe ../NEMS/exe/global_fv3gfs.x -fi + +./compile.sh "$target" "APP=ATM CCPP=Y 32BIT=Y SUITES=FV3_GFS_v16,FV3_GFS_v16_RRTMGP" 2 YES NO +mv -f fv3_2.exe ../NEMS/exe/global_fv3gfs.x diff --git a/sorc/build_ufs_utils.sh b/sorc/build_ufs_utils.sh index 292145d7645..0a22135a155 100755 --- a/sorc/build_ufs_utils.sh +++ b/sorc/build_ufs_utils.sh @@ -7,9 +7,9 @@ cwd=`pwd` if [ $target = wcoss_dell_p3 ]; then target=dell; fi if [ $target = wcoss_cray ]; then target=cray; fi -cd ufs_utils.fd/sorc +cd ufs_utils.fd -./build_all_ufs_utils.sh +./build_all.sh exit diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 32daff82a94..9ac4b97ea83 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -9,10 +9,6 @@ do echo "Received -o flag for optional checkout of GTG, will check out GTG with EMC_post" checkout_gtg="YES" ;; - c) - echo "Received -c flag, check out ufs-weather-model develop branch with CCPP physics" - run_ccpp="YES" - ;; :) echo "option -$OPTARG needs an argument" ;; @@ -31,11 +27,7 @@ if [[ ! -d fv3gfs.fd ]] ; then rm -f ${topdir}/checkout-fv3gfs.log git clone https://github.com/ufs-community/ufs-weather-model fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1 cd fv3gfs.fd - if [ ${run_ccpp:-"NO"} = "NO" ]; then - git checkout GFS.v16.0.16 - else - git checkout 9350745855aebe0790813e0ed2ba5ad680e3f75c - fi + git checkout 9350745855aebe0790813e0ed2ba5ad680e3f75c git submodule update --init --recursive cd ${topdir} else @@ -70,7 +62,7 @@ if [[ ! -d ufs_utils.fd ]] ; then rm -f ${topdir}/checkout-ufs_utils.log git clone https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1 cd ufs_utils.fd - git checkout ops-gfsv16.0.0 + git checkout ufs_utils_1_4_0 cd ${topdir} else echo 'Skip. Directory ufs_utils.fd already exists.' diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh index ae4494d83ef..fe823d2ee99 100755 --- a/sorc/link_fv3gfs.sh +++ b/sorc/link_fv3gfs.sh @@ -49,7 +49,7 @@ for dir in fix_aer fix_am fix_chem fix_fv3_gmted2010 fix_gldas fix_lut fix_orog done if [ -d ${pwd}/ufs_utils.fd ]; then - cd ${pwd}/ufs_utils.fd/sorc + cd ${pwd}/ufs_utils.fd/fix ./link_fixdirs.sh $RUN_ENVIR $machine fi @@ -77,8 +77,7 @@ cd ${pwd}/../ush ||exit 8 $LINK ../sorc/gfs_post.fd/ush/$file . done for file in emcsfc_ice_blend.sh fv3gfs_driver_grid.sh fv3gfs_make_orog.sh global_cycle_driver.sh \ - emcsfc_snow.sh fv3gfs_filter_topo.sh global_chgres_driver.sh global_cycle.sh \ - fv3gfs_chgres.sh fv3gfs_make_grid.sh global_chgres.sh ; do + emcsfc_snow.sh fv3gfs_filter_topo.sh global_cycle.sh fv3gfs_make_grid.sh ; do $LINK ../sorc/ufs_utils.fd/ush/$file . done for file in gldas_archive.sh gldas_forcing.sh gldas_get_data.sh gldas_process_data.sh gldas_liscrd.sh gldas_post.sh ; do @@ -234,7 +233,7 @@ if [ -d ${pwd}/gfs_wafs.fd ]; then fi for ufs_utilsexe in \ - emcsfc_ice_blend emcsfc_snow2mdl global_chgres global_cycle ; do + emcsfc_ice_blend emcsfc_snow2mdl global_cycle ; do [[ -s $ufs_utilsexe ]] && rm -f $ufs_utilsexe $LINK ../sorc/ufs_utils.fd/exec/$ufs_utilsexe . done @@ -310,13 +309,10 @@ cd ${pwd}/../sorc || exit 8 $SLINK gfs_post.fd/sorc/ncep_post.fd gfs_ncep_post.fd - $SLINK ufs_utils.fd/sorc/fre-nctools.fd/tools/shave.fd shave.fd - for prog in filter_topo fregrid make_hgrid make_solo_mosaic ; do + for prog in fregrid make_hgrid make_solo_mosaic ; do $SLINK ufs_utils.fd/sorc/fre-nctools.fd/tools/$prog ${prog}.fd done - for prog in global_cycle.fd nemsio_read.fd nemsio_chgdate.fd \ - emcsfc_ice_blend.fd nst_tf_chg.fd \ - emcsfc_snow2mdl.fd global_chgres.fd nemsio_get.fd orog.fd ;do + for prog in global_cycle.fd emcsfc_ice_blend.fd emcsfc_snow2mdl.fd ; do $SLINK ufs_utils.fd/sorc/$prog $prog done diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index c39cbfb2722..5f28d6abaa9 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -45,11 +45,10 @@ elif [[ -d /scratch1 ]] ; then fi target=hera module purge - module load intel - module load impi - export NCEPLIBS=/scratch2/NCEPDEV/nwprod/NCEPLIBS - module use $NCEPLIBS/modulefiles - #export WRFPATH=$NCEPLIBS/wrf.shared.new/v1.1.1/src + module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack + module load hpc/1.1.0 + module load cmake/3.20.0 + export myFC=mpiifort export FCOMP=mpiifort @@ -95,6 +94,9 @@ elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then fi target=wcoss_dell_p3 module purge + module use /gpfs/dell2/usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles + module load hpc/1.1.0 + module load cmake/3.20.0 ##--------------------------------------------------------------------------- diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh index 1f4c8ed542b..4e65d7dbdf7 100755 --- a/ush/hpssarch_gen.sh +++ b/ush/hpssarch_gen.sh @@ -41,17 +41,23 @@ if [ $type = "gfs" ]; then rm -f gfsb.txt rm -f gfs_pgrb2b.txt rm -f gfs_flux.txt - rm -f gfs_${format}a.txt - rm -f gfs_${format}b.txt rm -f gfs_restarta.txt touch gfsa.txt touch gfsb.txt touch gfs_pgrb2b.txt touch gfs_flux.txt - touch gfs_${format}a.txt - touch gfs_${format}b.txt touch gfs_restarta.txt + if [ $MODE = "cycled" ]; then + rm -f gfs_${format}a.txt + touch gfs_${format}a.txt + fi + + if [ $OUTPUT_HISTORY = ".true." ]; then + rm -f gfs_${format}b.txt + touch gfs_${format}b.txt + fi + if [ $DO_DOWN = "YES" ]; then rm -f gfs_downstream.txt touch gfs_downstream.txt @@ -69,11 +75,13 @@ if [ $type = "gfs" ]; then echo "${dirname}${head}pgrb2b.0p50.anl.idx " >>gfs_pgrb2b.txt echo "./logs/${CDATE}/gfs*.log " >>gfsa.txt - echo "${dirname}${head}gsistat " >>gfsa.txt - echo "${dirname}${head}nsstbufr " >>gfsa.txt - echo "${dirname}${head}prepbufr " >>gfsa.txt - echo "${dirname}${head}prepbufr_pre-qc " >>gfsa.txt - echo "${dirname}${head}prepbufr.acft_profiles " >>gfsa.txt + if [ $MODE = "cycled" ]; then + echo "${dirname}${head}gsistat " >>gfsa.txt + echo "${dirname}${head}nsstbufr " >>gfsa.txt + echo "${dirname}${head}prepbufr " >>gfsa.txt + echo "${dirname}${head}prepbufr_pre-qc " >>gfsa.txt + echo "${dirname}${head}prepbufr.acft_profiles " >>gfsa.txt + fi echo "${dirname}${head}pgrb2.0p25.anl " >>gfsa.txt echo "${dirname}${head}pgrb2.0p25.anl.idx " >>gfsa.txt echo "${dirname}avno.t${cyc}z.cyclone.trackatcfunix " >>gfsa.txt @@ -145,11 +153,13 @@ if [ $type = "gfs" ]; then #.................. - echo "${dirname}${head}atmanl${SUFFIX} " >>gfs_${format}a.txt - echo "${dirname}${head}sfcanl${SUFFIX} " >>gfs_${format}a.txt - echo "${dirname}${head}atmi*.nc " >>gfs_${format}a.txt - echo "${dirname}${head}dtfanl.nc " >>gfs_${format}a.txt - echo "${dirname}${head}loginc.txt " >>gfs_${format}a.txt + if [ $MODE = "cycled" ]; then + echo "${dirname}${head}atmanl${SUFFIX} " >>gfs_${format}a.txt + echo "${dirname}${head}sfcanl${SUFFIX} " >>gfs_${format}a.txt + echo "${dirname}${head}atmi*.nc " >>gfs_${format}a.txt + echo "${dirname}${head}dtfanl.nc " >>gfs_${format}a.txt + echo "${dirname}${head}loginc.txt " >>gfs_${format}a.txt + fi #.................. if [ $OUTPUT_HISTORY = ".true." ]; then @@ -163,12 +173,28 @@ if [ $type = "gfs" ]; then fi #.................. - echo "${dirname}RESTART/*0000.sfcanl_data.tile1.nc " >>gfs_restarta.txt - echo "${dirname}RESTART/*0000.sfcanl_data.tile2.nc " >>gfs_restarta.txt - echo "${dirname}RESTART/*0000.sfcanl_data.tile3.nc " >>gfs_restarta.txt - echo "${dirname}RESTART/*0000.sfcanl_data.tile4.nc " >>gfs_restarta.txt - echo "${dirname}RESTART/*0000.sfcanl_data.tile5.nc " >>gfs_restarta.txt - echo "${dirname}RESTART/*0000.sfcanl_data.tile6.nc " >>gfs_restarta.txt + if [ $MODE = "cycled" ]; then + echo "${dirname}RESTART/*0000.sfcanl_data.tile1.nc " >>gfs_restarta.txt + echo "${dirname}RESTART/*0000.sfcanl_data.tile2.nc " >>gfs_restarta.txt + echo "${dirname}RESTART/*0000.sfcanl_data.tile3.nc " >>gfs_restarta.txt + echo "${dirname}RESTART/*0000.sfcanl_data.tile4.nc " >>gfs_restarta.txt + echo "${dirname}RESTART/*0000.sfcanl_data.tile5.nc " >>gfs_restarta.txt + echo "${dirname}RESTART/*0000.sfcanl_data.tile6.nc " >>gfs_restarta.txt + elif [ $MODE = "free" ]; then + echo "${dirname}INPUT/gfs_ctrl.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/gfs_data.tile1.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/gfs_data.tile2.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/gfs_data.tile3.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/gfs_data.tile4.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/gfs_data.tile5.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/gfs_data.tile6.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/sfc_data.tile1.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/sfc_data.tile2.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/sfc_data.tile3.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/sfc_data.tile4.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/sfc_data.tile5.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/sfc_data.tile6.nc " >>gfs_restarta.txt + fi #.................. if [ $DO_WAVE = "YES" ]; then diff --git a/ush/rocoto/setup_expt.py b/ush/rocoto/setup_expt.py index 2496ef9dc36..6a4b161bc1f 100755 --- a/ush/rocoto/setup_expt.py +++ b/ush/rocoto/setup_expt.py @@ -105,6 +105,7 @@ def edit_baseconfig(): .replace('@QUEUE_SERVICE@', queue_service) \ .replace('@PARTITION_BATCH@', partition_batch) \ .replace('@EXP_WARM_START@', exp_warm_start) \ + .replace('@MODE@', 'cycled') \ .replace('@CHGRP_RSTPROD@', chgrp_rstprod) \ .replace('@CHGRP_CMD@', chgrp_cmd) \ .replace('@HPSSARCH@', hpssarch) \ diff --git a/ush/rocoto/setup_expt_fcstonly.py b/ush/rocoto/setup_expt_fcstonly.py index 3a57a07bea3..f556683b934 100755 --- a/ush/rocoto/setup_expt_fcstonly.py +++ b/ush/rocoto/setup_expt_fcstonly.py @@ -79,6 +79,7 @@ def edit_baseconfig(): .replace('@QUEUE_SERVICE@', queue_service) \ .replace('@PARTITION_BATCH@', partition_batch) \ .replace('@EXP_WARM_START@', exp_warm_start) \ + .replace('@MODE@', 'free') \ .replace('@CHGRP_RSTPROD@', chgrp_rstprod) \ .replace('@CHGRP_CMD@', chgrp_cmd) \ .replace('@HPSSARCH@', hpssarch) \ diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index 4a4bfa6bb82..f20a09a2668 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -161,6 +161,7 @@ def get_definitions(base): machine = base.get('machine', wfu.detectMachine()) scheduler = wfu.get_scheduler(machine) + hpssarch = base.get('HPSSARCH', 'NO').upper() strings = [] @@ -198,7 +199,7 @@ def get_definitions(base): strings.append('\t\n' % scheduler) strings.append('\n') strings.append('\t\n') - strings.append('\t\n') + strings.append('\t\n' % base['HPSSARCH']) strings.append('\n') strings.append('\t\n') strings.append('\t\n') @@ -918,8 +919,6 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): deps = [] dep_dict = {'type': 'task', 'name': '%svrfy' % cdump} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'streq', 'left': '&ARCHIVE_TO_HPSS;', 'right': 'YES'} - deps.append(rocoto.add_dependency(dep_dict)) if do_wave in ['Y', 'YES']: dep_dict = {'type': 'task', 'name': '%swavepostsbs' % cdump} deps.append(rocoto.add_dependency(dep_dict)) diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index 4d39cc7c12a..f25ec53f1c5 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -27,8 +27,7 @@ import rocoto import workflow_utils as wfu -#taskplan = ['getic', 'fv3ic', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wavepost', 'wavestat', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'vrfy', 'metp', 'arch'] -taskplan = ['getic', 'fv3ic', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'vrfy', 'metp', 'arch'] +taskplan = ['getic', 'init', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'postsnd', 'gempak', 'awips', 'vrfy', 'metp', 'arch'] def main(): parser = ArgumentParser(description='Setup XML workflow and CRONTAB for a forecast only experiment.', formatter_class=ArgumentDefaultsHelpFormatter) @@ -92,6 +91,7 @@ def get_definitions(base): machine = base.get('machine', wfu.detectMachine()) scheduler = wfu.get_scheduler(machine) + hpssarch = base.get('HPSSARCH', 'NO').upper() strings = [] @@ -132,7 +132,7 @@ def get_definitions(base): strings.append('\t\n' % scheduler) strings.append('\n') strings.append('\t\n') - strings.append('\t\n') + strings.append('\t\n' % base['HPSSARCH']) strings.append('\n') strings.append('\t\n') strings.append('\t\n') @@ -159,6 +159,7 @@ def get_resources(dict_configs, cdump='gdas'): scheduler = wfu.get_scheduler(machine) do_wave = base.get('DO_WAVE', 'NO').upper() + do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() do_gempak = base.get('DO_GEMPAK', 'NO').upper() do_awips = base.get('DO_AWIPS', 'NO').upper() do_metp = base.get('DO_METP', 'NO').upper() @@ -235,8 +236,11 @@ def get_workflow(dict_configs, cdump='gdas'): envars.append(rocoto.create_envar(name='cyc', value='@H')) base = dict_configs['base'] + machine = base.get('machine', wfu.detectMachine()) + hpssarch = base.get('HPSSARCH', 'NO').upper() do_wave = base.get('DO_WAVE', 'NO').upper() do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() + do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() do_gempak = base.get('DO_GEMPAK', 'NO').upper() do_awips = base.get('DO_AWIPS', 'NO').upper() do_wafs = base.get('WAFSF', 'NO').upper() @@ -244,73 +248,53 @@ def get_workflow(dict_configs, cdump='gdas'): tasks = [] - # getics + # getic + if hpssarch in ['YES']: + deps = [] + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' + dep_dict = {'type':'data', 'data':data} + deps.append(rocoto.add_dependency(dep_dict)) + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' + dep_dict = {'type':'data', 'data':data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) + + task = wfu.create_wf_task('getic', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # init deps = [] - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/siganl.&CDUMP;.@Y@m@d@H' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.sanl' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;.t@Hz.sanl' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nemsio' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/gfnanl.&CDUMP;.@Y@m@d@H' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;.t@Hz.atmanl.nemsio' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/gfs.t@Hz.atmanl.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - - deps = [] - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/gfs_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - deps = rocoto.create_dependency(dep_condition='and', dep=deps) - dependencies2 = rocoto.create_dependency(dep_condition='not', dep=deps) - - deps = [] - deps.append(dependencies) - deps.append(dependencies2) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - - task = wfu.create_wf_task('getic', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # chgres fv3ic - deps = [] - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/siganl.&CDUMP;.@Y@m@d@H' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;.t@Hz.sanl' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/gfnanl.&CDUMP;.@Y@m@d@H' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;.t@Hz.atmanl.nemsio' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - deps = [] - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/gfs_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - deps = rocoto.create_dependency(dep_condition='and', dep=deps) - dependencies2 = rocoto.create_dependency(dep_condition='not', dep=deps) + if hpssarch in ['YES']: + deps = [] + dep_dict = {'type': 'task', 'name': '%sgetic' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies2 = rocoto.create_dependency(dep=deps) deps = [] deps.append(dependencies) - deps.append(dependencies2) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + if hpssarch in ['YES']: + deps.append(dependencies2) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('fv3ic', cdump=cdump, envar=envars, dependency=dependencies) + task = wfu.create_wf_task('init', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') @@ -325,17 +309,19 @@ def get_workflow(dict_configs, cdump='gdas'): deps = [] dep_dict = {'type': 'task', 'name': '%swaveinit' % cdump} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + dep_dict = {'type': 'task', 'name': '%sinit' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('waveprep', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') # fcst deps = [] - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) @@ -424,16 +410,6 @@ def get_workflow(dict_configs, cdump='gdas'): tasks.append(task) tasks.append('\n') - # wavestat - #if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - # deps = [] - # dep_dict = {'type':'task', 'name':'%swavepost' % cdump} - # deps.append(rocoto.add_dependency(dep_dict)) - # dependencies = rocoto.create_dependency(dep=deps) - # task = wfu.create_wf_task('wavestat', cdump=cdump, envar=envars, dependency=dependencies) - # tasks.append(task) - # tasks.append('\n') - # wavegempak if do_wave in ['Y', 'YES'] and do_gempak in ['Y', 'YES']: deps = [] @@ -638,6 +614,44 @@ def get_workflow(dict_configs, cdump='gdas'): tasks.append(task) tasks.append('\n') + #postsnd + if do_bufrsnd in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'task', 'name': '%sfcst' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('postsnd', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # awips + if do_awips in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'metatask', 'name': '%spost' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') + fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') + ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') + awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'], cdump=cdump) + vardict = {varname2: varval2, varname3: varval3} + task = wfu.create_wf_task('awips', cdump=cdump, envar=awipsenvars, dependency=dependencies, + metatask='awips', varname=varname1, varval=varval1, vardict=vardict) + tasks.append(task) + tasks.append('\n') + + # gempak + if do_gempak in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'metatask', 'name': '%spost' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('gempak', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + # vrfy deps = [] dep_dict = {'type':'metatask', 'name':'%spost' % cdump} @@ -671,8 +685,13 @@ def get_workflow(dict_configs, cdump='gdas'): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type':'task', 'name':'%svrfy' % cdump} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type':'streq', 'left':'&ARCHIVE_TO_HPSS;', 'right':'YES'} - deps.append(rocoto.add_dependency(dep_dict)) + if do_wave in ['Y', 'YES']: + dep_dict = {'type': 'task', 'name': '%swavepostsbs' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': '%swavepostpnt' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': '%swavepostbndpnt' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies, final=True) tasks.append(task)