From a7391d70716c5a543ec57cf74f2c5cf133c15330 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 28 Jan 2021 19:04:52 +0000 Subject: [PATCH 01/40] Update free-forecast mode chgres jobs for chgres_cube Update free-forecast mode to interface with UFS_UTILS gdas_init utility scripts. Update getic job to use gdas_init get scripts to pull ICs off HPSS for GFS versions 13 and later. Rename fv3ic job to "init" and update it to interface with gdas_init run scripts to run chgres_cube and produce GFSv16 ICs. Update job dependencies to detect need to run chgres jobs and hold forecast jobs until ICs are generated or present. Further updates coming for this task. Tested on WCOSS-Dell, need to test elsewhere still. Will disable getic job on Orion. Refs: #1, #178 --- env/HERA.env | 6 +- env/JET.env | 6 +- env/ORION.env | 6 +- env/WCOSS_C.env | 6 +- env/WCOSS_DELL_P3.env | 6 +- jobs/rocoto/fv3ic.sh | 69 --------- jobs/rocoto/getic.sh | 212 +++++++------------------- jobs/rocoto/init.sh | 81 ++++++++++ parm/config/config.getic | 10 +- parm/config/config.init | 19 +++ parm/config/config.resources | 7 + sorc/build_ufs_utils.sh | 4 +- sorc/link_fv3gfs.sh | 2 +- ush/rocoto/setup_workflow_fcstonly.py | 59 ++----- 14 files changed, 193 insertions(+), 300 deletions(-) delete mode 100755 jobs/rocoto/fv3ic.sh create mode 100755 jobs/rocoto/init.sh create mode 100755 parm/config/config.init diff --git a/env/HERA.env b/env/HERA.env index f71531c1ed2..4290bdb259b 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -214,11 +214,9 @@ elif [ $step = "epos" ]; then [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max export APRUN_EPOS="$launcher" -elif [ $step = "fv3ic" ]; then +elif [ $step = "init" ]; then - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" + export APRUN="$launcher" elif [ $step = "postsnd" ]; then diff --git a/env/JET.env b/env/JET.env index ef37e468df0..5bbb903bd11 100755 --- a/env/JET.env +++ b/env/JET.env @@ -179,11 +179,9 @@ elif [ $step = "epos" ]; then [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max export APRUN_EPOS="$launcher ${npe_epos:-$PBS_NP}" -elif [ $step = "fv3ic" ]; then +elif [ $step = "init" ]; then - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" + export APRUN="$launcher" elif [ $step = "postsnd" ]; then diff --git a/env/ORION.env b/env/ORION.env index de989331ff1..69721c21edc 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -202,11 +202,9 @@ elif [ $step = "epos" ]; then [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max export APRUN_EPOS="$launcher" -elif [ $step = "fv3ic" ]; then +elif [ $step = "init" ]; then - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" + export APRUN="$launcher" elif [ $step = "postsnd" ]; then diff --git a/env/WCOSS_C.env b/env/WCOSS_C.env index 4885a966a05..bd88a44f31a 100755 --- a/env/WCOSS_C.env +++ b/env/WCOSS_C.env @@ -198,11 +198,9 @@ elif [ $step = "epos" ]; then [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max export APRUN_EPOS="$launcher -j 1 -n $npe_epos -N $npe_node_epos -d $NTHREADS_EPOS -cc depth" -elif [ $step = "fv3ic" ]; then +elif [ $step = "init" ]; then - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="$launcher -j 1 -n 1 -N 1 -d $NTHREADS_CHGRES -cc depth" + export APRUN="$launcher" elif [ $step = "vrfy" ]; then diff --git a/env/WCOSS_DELL_P3.env b/env/WCOSS_DELL_P3.env index a028826b931..e2a6989f1d7 100755 --- a/env/WCOSS_DELL_P3.env +++ b/env/WCOSS_DELL_P3.env @@ -204,11 +204,9 @@ elif [ $step = "epos" ]; then [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max export APRUN_EPOS="$launcher ${npe_epos:-$PBS_NP}" -elif [ $step = "fv3ic" ]; then +elif [ $step = "init" ]; then - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" + export APRUN="mpirun" elif [ $step = "postsnd" ]; then diff --git a/jobs/rocoto/fv3ic.sh b/jobs/rocoto/fv3ic.sh deleted file mode 100755 index b5c2e40b484..00000000000 --- a/jobs/rocoto/fv3ic.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/ksh -x - -############################################################### -## Abstract: -## Create FV3 initial conditions from GFS intitial conditions -## RUN_ENVIR : runtime environment (emc | nco) -## HOMEgfs : /full/path/to/workflow -## EXPDIR : /full/path/to/config/files -## CDATE : current date (YYYYMMDDHH) -## CDUMP : cycle name (gdas / gfs) -## PDY : current date (YYYYMMDD) -## cyc : current cycle (HH) -############################################################### - -############################################################### -# Source FV3GFS workflow modules -. $HOMEgfs/ush/load_fv3gfs_modules.sh -status=$? -[[ $status -ne 0 ]] && exit $status - -############################################################### -# Source relevant configs -configs="base fv3ic" -for config in $configs; do - . $EXPDIR/config.${config} - status=$? - [[ $status -ne 0 ]] && exit $status -done - -############################################################### -# Source machine runtime environment -. $BASE_ENV/${machine}.env fv3ic -status=$? -[[ $status -ne 0 ]] && exit $status - -# Set component -export COMPONENT=${COMPONENT:-atmos} - -# Temporary runtime directory -export DATA="$RUNDIR/$CDATE/$CDUMP/fv3ic$$" -[[ -d $DATA ]] && rm -rf $DATA - -# Input GFS initial condition directory -export INIDIR="$ICSDIR/$CDATE/$CDUMP/$CDUMP.$PDY/$cyc" - -# Output FV3 initial condition directory -export OUTDIR="$ICSDIR/$CDATE/$CDUMP/$CASE/INPUT" - -export OMP_NUM_THREADS_CH=$NTHREADS_CHGRES -export APRUNC=$APRUN_CHGRES - -# Call global_chgres_driver.sh -$HOMEgfs/ush/global_chgres_driver.sh -status=$? -if [ $status -ne 0 ]; then - echo "global_chgres_driver.sh returned with a non-zero exit code, ABORT!" - exit $status -fi - -# Stage the FV3 initial conditions to ROTDIR -COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT" -[[ ! -d $COMOUT ]] && mkdir -p $COMOUT -cd $COMOUT || exit 99 -rm -rf INPUT -$NLN $OUTDIR . - -############################################################### -# Exit cleanly -exit 0 diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index 120e3b9454f..fd07a9ead4f 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -36,166 +36,68 @@ status=$? ############################################################### # Set script and dependency variables -yyyy=$(echo $CDATE | cut -c1-4) +yy=$(echo $CDATE | cut -c1-4) mm=$(echo $CDATE | cut -c5-6) dd=$(echo $CDATE | cut -c7-8) -cyc=${cyc:-$(echo $CDATE | cut -c9-10)} - -export COMPONENT=${COMPONENT:-atmos} - -############################################################### - -target_dir=$ICSDIR/$CDATE/$CDUMP -mkdir -p $target_dir -cd $target_dir - -# Initialize return code to 0 -rc=1 - -if [ $ics_from = "opsgfs" ]; then - - # Location of production tarballs on HPSS - hpssdir="/NCEPPROD/hpssprod/runhistory/rh$yyyy/$yyyy$mm/$PDY" - - # Handle nemsio and pre-nemsio GFS filenames - if [ $CDATE -le "2019061118" ]; then #GFSv14 - # Add CDUMP.PDY/CYC to target_dir - target_dir=$ICSDIR/$CDATE/$CDUMP/${CDUMP}.$yyyy$mm$dd/$cyc - mkdir -p $target_dir - cd $target_dir - - nfanal=4 - fanal[1]="./${CDUMP}.t${cyc}z.atmanl.nemsio" - fanal[2]="./${CDUMP}.t${cyc}z.sfcanl.nemsio" - fanal[3]="./${CDUMP}.t${cyc}z.nstanl.nemsio" - fanal[4]="./${CDUMP}.t${cyc}z.pgrbanl" - flanal="${fanal[1]} ${fanal[2]} ${fanal[3]} ${fanal[4]}" - tarpref="gpfs_hps_nco_ops_com" - if [ $CDUMP = "gdas" ]; then - tarball="$hpssdir/${tarpref}_gfs_prod_${CDUMP}.${CDATE}.tar" - elif [ $CDUMP = "gfs" ]; then - tarball="$hpssdir/${tarpref}_gfs_prod_${CDUMP}.${CDATE}.anl.tar" - fi - else #GFSv15 - nfanal=2 - fanal[1]="./${CDUMP}.$yyyy$mm$dd/$cyc/${CDUMP}.t${cyc}z.atmanl.nemsio" - fanal[2]="./${CDUMP}.$yyyy$mm$dd/$cyc/${CDUMP}.t${cyc}z.sfcanl.nemsio" - flanal="${fanal[1]} ${fanal[2]}" - if [ $CDATE -ge "2020022600" ]; then - tarpref="com" - else - tarpref="gpfs_dell1_nco_ops_com" - fi - if [ $CDUMP = "gdas" ]; then - tarball="$hpssdir/${tarpref}_gfs_prod_${CDUMP}.${yyyy}${mm}${dd}_${cyc}.${CDUMP}_nemsio.tar" - elif [ $CDUMP = "gfs" ]; then - tarball="$hpssdir/${tarpref}_gfs_prod_${CDUMP}.${yyyy}${mm}${dd}_${cyc}.${CDUMP}_nemsioa.tar" - fi - fi - - # First check the COMROOT for files, if present copy over - if [ $machine = "WCOSS_C" ]; then - - # Need COMROOT - module load prod_envir/1.1.0 >> /dev/null 2>&1 - - comdir="$COMROOT/$CDUMP/prod/$CDUMP.$PDY" - rc=0 - for i in `seq 1 $nfanal`; do - if [ -f $comdir/${fanal[i]} ]; then - $NCP $comdir/${fanal[i]} ${fanal[i]} - else - rb=1 ; ((rc+=rb)) - fi - done - - fi - - # Get initial conditions from HPSS - if [ $rc -ne 0 ]; then - - # check if the tarball exists - hsi ls -l $tarball - rc=$? - if [ $rc -ne 0 ]; then - echo "$tarball does not exist and should, ABORT!" - exit $rc - fi - # get the tarball - htar -xvf $tarball $flanal - rc=$? - if [ $rc -ne 0 ]; then - echo "untarring $tarball failed, ABORT!" - exit $rc - fi - - # Move the files to legacy EMC filenames - if [ $CDATE -le "2019061118" ]; then #GFSv14 - for i in `seq 1 $nfanal`; do - $NMV ${fanal[i]} ${flanal[i]} - done - fi - - fi - - # If found, exit out - if [ $rc -ne 0 ]; then - echo "Unable to obtain operational GFS initial conditions, ABORT!" - exit 1 - fi - -elif [ $ics_from = "pargfs" ]; then - - # Add CDUMP.PDY/CYC to target_dir - target_dir=$ICSDIR/$CDATE/$CDUMP/${CDUMP}.$yyyy$mm$dd/$cyc - mkdir -p $target_dir - cd $target_dir - - # Filenames in parallel - nfanal=4 - fanal[1]="gfnanl.${CDUMP}.$CDATE" - fanal[2]="sfnanl.${CDUMP}.$CDATE" - fanal[3]="nsnanl.${CDUMP}.$CDATE" - fanal[4]="pgbanl.${CDUMP}.$CDATE" - flanal="${fanal[1]} ${fanal[2]} ${fanal[3]} ${fanal[4]}" - - # Get initial conditions from HPSS from retrospective parallel - tarball="$HPSS_PAR_PATH/${CDATE}${CDUMP}.tar" - - # check if the tarball exists - hsi ls -l $tarball - rc=$? - if [ $rc -ne 0 ]; then - echo "$tarball does not exist and should, ABORT!" - exit $rc - fi - # get the tarball - htar -xvf $tarball $flanal - rc=$? - if [ $rc -ne 0 ]; then - echo "untarring $tarball failed, ABORT!" - exit $rc - fi - - # If found, exit out - if [ $rc -ne 0 ]; then - echo "Unable to obtain parallel GFS initial conditions, ABORT!" - exit 1 - fi - -else +hh=${cyc:-$(echo $CDATE | cut -c9-10)} + +EXTRACT_DIR=${PTMP}/gdas.init_${CDATE}/input +OUTDIR=${PTMP}/gdas.init_${CDATE}/output +PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory/rh${yy}/${yy}${mm}/${yy}${mm}${dd} + +COMPONENT="atmos" + +gfs_ver=v16 +GETICSH=${GDASINIT_DIR}/get_v16.data.sh + +# No ENKF data prior to 2012/05/21/00z +if [ $yy$mm$dd$hh -lt 2012052100 ]; then + set +x + echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA + exit 2 +elif [ $yy$mm$dd$hh -lt 2016051000 ]; then + gfs_ver=v12 + GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh +elif [ $yy$mm$dd$hh -lt 2017072000 ]; then + gfs_ver=v13 + GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh +elif [ $yy$mm$dd$hh -lt 2019061200 ]; then + gfs_ver=v14 + GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh +elif [ $yy$mm$dd$hh -lt 2021020300 ]; then + gfs_ver=v15 + GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh +fi - echo "ics_from = $ics_from is not supported, ABORT!" - exit 1 +export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF +export LEVS gfs_ver -fi -############################################################### +# Run get data script +if [ ! -d $EXTRACT_DIR ]; then mkdir -p $EXTRACT_DIR ; fi +sh ${GETICSH} ${CDUMP} +status=$? +[[ $status -ne 0 ]] && exit $status -# Copy pgbanl file to COMROT for verification - GFSv14 only -if [ $CDATE -le "2019061118" ]; then #GFSv14 - COMROT=$ROTDIR/${CDUMP}.$PDY/$cyc/$COMPONENT - [[ ! -d $COMROT ]] && mkdir -p $COMROT - $NCP ${fanal[4]} $COMROT/${CDUMP}.t${cyc}z.pgrbanl +# Copy pgbanl file to ROTDIR for verification/archival - v14+ +cd $EXTRACT_DIR +OUTDIR2=${ROTDIR}/gfs.${yy}${mm}${dd}/${hh}/${COMPONENT} +if [ ! -d ${OUTDIR2} ]; then mkdir -p ${OUTDIR2} ; fi +if [ $gfs_ver = v14 ]; then + for grid in 0p25 0p50 1p00 + do + tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.pgrb2_${grid}.tar + file=gfs.t${hh}z.pgrb2.${grid}.anl + htar -xvf ${PRODHPSSDIR}/${tarball} ./gfs.${yy}${mm}${dd}/${hh}/${file} + mv ${EXTRACT_DIR}/gfs.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR2}/${file} + done +elif [ $gfs_ver = v15 ]; then + tarball=com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar + for grid in 0p25 0p50 1p00 + do + file=gfs.t${hh}z.pgrb2.${grid}.anl + htar -xvf ${PRODHPSSDIR}/${tarball} ./gfs.${yy}${mm}${dd}/${hh}/${file} + mv ${EXTRACT_DIR}/gfs.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR2}/${file} + done fi ############################################################### diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh new file mode 100755 index 00000000000..c36addb2ed5 --- /dev/null +++ b/jobs/rocoto/init.sh @@ -0,0 +1,81 @@ +#!/bin/ksh -x + +############################################################### +## Abstract: +## Get GFS intitial conditions +## RUN_ENVIR : runtime environment (emc | nco) +## HOMEgfs : /full/path/to/workflow +## EXPDIR : /full/path/to/config/files +## CDATE : current date (YYYYMMDDHH) +## CDUMP : cycle name (gdas / gfs) +## PDY : current date (YYYYMMDD) +## cyc : current cycle (HH) +############################################################### + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Source relevant configs +configs="base init" +for config in $configs; do + . $EXPDIR/config.${config} + status=$? + [[ $status -ne 0 ]] && exit $status +done + +############################################################### +# Source machine runtime environment +. $BASE_ENV/${machine}.env init +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Set script and dependency variables + +yy=$(echo $CDATE | cut -c1-4) +mm=$(echo $CDATE | cut -c5-6) +dd=$(echo $CDATE | cut -c7-8) +hh=${cyc:-$(echo $CDATE | cut -c9-10)} + +EXTRACT_DIR=${PTMP}/gdas.init_${CDATE}/input +WORKDIR=${PTMP}/gdas.init_${CDATE}/output +OUTDIR=${ROTDIR} + +gfs_ver=v16 +RUNICSH=${GDASINIT_DIR}/run_v16.chgres.sh + +# No ENKF data prior to 2012/05/21/00z +if [ $yy$mm$dd$hh -lt 2012052100 ]; then + set +x + echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA + exit 2 +elif [ $yy$mm$dd$hh -lt 2016051000 ]; then + gfs_ver=v12 + RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh +elif [ $yy$mm$dd$hh -lt 2017072000 ]; then + gfs_ver=v13 + RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh +elif [ $yy$mm$dd$hh -lt 2019061200 ]; then + gfs_ver=v14 + RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh +elif [ $yy$mm$dd$hh -lt 2021020300 ]; then + gfs_ver=v15 + RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh +fi + +export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF +export LEVS gfs_ver + +# Run chgres_cube +if [ ! -d $OUTDIR ]; then mkdir -p $OUTDIR ; fi +sh ${RUNICSH} ${CDUMP} +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Exit out cleanly +exit 0 diff --git a/parm/config/config.getic b/parm/config/config.getic index a7544542641..64f7159cc98 100755 --- a/parm/config/config.getic +++ b/parm/config/config.getic @@ -8,13 +8,9 @@ echo "BEGIN: config.getic" # Get task specific resources . $EXPDIR/config.resources getic -# We should just be supporting the OPSGFS only -export ics_from="opsgfs" # initial conditions from opsgfs or pargfs +UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd +GDASINIT_DIR=${UFS_DIR}/util/gdas_init -# Provide a parallel experiment name and path to HPSS archive -if [ $ics_from = "pargfs" ]; then - export parexp="prnemsrn" - export HPSS_PAR_PATH="/5year/NCEPDEV/emc-global/emc.glopara/WCOSS_C/$parexp" -fi +EXTRACT_DATA="yes" echo "END: config.getic" diff --git a/parm/config/config.init b/parm/config/config.init new file mode 100755 index 00000000000..f9efbde888b --- /dev/null +++ b/parm/config/config.init @@ -0,0 +1,19 @@ +#!/bin/ksh -x + +########## config.init ########## +# Prepare initial conditions + +echo "BEGIN: config.init" + +# Get task specific resources +. $EXPDIR/config.resources init + +UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd +GDASINIT_DIR=${UFS_DIR}/util/gdas_init + +RUN_CHGRES="yes" + +CRES_HIRES=$CASE +CRES_ENKF=$CASE_ENKF + +echo "END: config.init" diff --git a/parm/config/config.resources b/parm/config/config.resources index 3d996d96b65..d62424aa324 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -262,6 +262,13 @@ elif [ $step = "echgres" ]; then export nth_echgres=$npe_node_max export npe_node_echgres=1 +elif [ $step = "init" ]; then + + export wtime_init="00:15:00" + export npe_init=24 + export nth_init=1 + export npe_node_init=6 + elif [ $step = "arch" -o $step = "earc" -o $step = "getic" ]; then eval "export wtime_$step='06:00:00'" diff --git a/sorc/build_ufs_utils.sh b/sorc/build_ufs_utils.sh index 292145d7645..0a22135a155 100755 --- a/sorc/build_ufs_utils.sh +++ b/sorc/build_ufs_utils.sh @@ -7,9 +7,9 @@ cwd=`pwd` if [ $target = wcoss_dell_p3 ]; then target=dell; fi if [ $target = wcoss_cray ]; then target=cray; fi -cd ufs_utils.fd/sorc +cd ufs_utils.fd -./build_all_ufs_utils.sh +./build_all.sh exit diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh index e5f0f08ad44..403c7364cae 100755 --- a/sorc/link_fv3gfs.sh +++ b/sorc/link_fv3gfs.sh @@ -49,7 +49,7 @@ for dir in fix_am fix_fv3_gmted2010 fix_gldas fix_orog fix_verif fix_wave_gfs ; done if [ -d ${pwd}/ufs_utils.fd ]; then - cd ${pwd}/ufs_utils.fd/sorc + cd ${pwd}/ufs_utils.fd/fix ./link_fixdirs.sh $RUN_ENVIR $machine fi diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index c7021f904e5..795602204eb 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -27,8 +27,7 @@ import rocoto import workflow_utils as wfu -#taskplan = ['getic', 'fv3ic', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wavepost', 'wavestat', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'vrfy', 'metp', 'arch'] -taskplan = ['getic', 'fv3ic', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'vrfy', 'metp', 'arch'] +taskplan = ['getic', 'init', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'vrfy', 'metp', 'arch'] def main(): parser = ArgumentParser(description='Setup XML workflow and CRONTAB for a forecast only experiment.', formatter_class=ArgumentDefaultsHelpFormatter) @@ -244,73 +243,41 @@ def get_workflow(dict_configs, cdump='gdas'): tasks = [] - # getics + # getic deps = [] - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/siganl.&CDUMP;.@Y@m@d@H' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/INPUT/sfc_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;.t@Hz.sanl' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/gfnanl.&CDUMP;.@Y@m@d@H' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;.t@Hz.atmanl.nemsio' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - - deps = [] - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/gfs_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - deps = rocoto.create_dependency(dep_condition='and', dep=deps) - dependencies2 = rocoto.create_dependency(dep_condition='not', dep=deps) - - deps = [] - deps.append(dependencies) - deps.append(dependencies2) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) task = wfu.create_wf_task('getic', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') - # chgres fv3ic + # init - chgres_cube deps = [] - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/siganl.&CDUMP;.@Y@m@d@H' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/INPUT/sfc_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;.t@Hz.sanl' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/gfnanl.&CDUMP;.@Y@m@d@H' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;.t@Hz.atmanl.nemsio' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) + dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) deps = [] - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/gfs_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} + dep_dict = {'type': 'task', 'name': '%sgetic' % cdump} deps.append(rocoto.add_dependency(dep_dict)) - deps = rocoto.create_dependency(dep_condition='and', dep=deps) - dependencies2 = rocoto.create_dependency(dep_condition='not', dep=deps) + dependencies2 = rocoto.create_dependency(dep=deps) deps = [] deps.append(dependencies) deps.append(dependencies2) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('fv3ic', cdump=cdump, envar=envars, dependency=dependencies) + task = wfu.create_wf_task('init', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') @@ -332,7 +299,7 @@ def get_workflow(dict_configs, cdump='gdas'): # fcst deps = [] - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/INPUT/sfc_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' From b251164cb3adfab274d7f7ea15c30eaff5eabf79 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 29 Jan 2021 19:45:44 +0000 Subject: [PATCH 02/40] Add missing component subfolder to fcst dependency The free-forecast mode fcst job data dependency file path was missing the component "atmos" subfolder. Added and tested in experiment on Hera. Refs: #178 --- ush/rocoto/setup_workflow_fcstonly.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index 795602204eb..1bd02a8504a 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -299,10 +299,10 @@ def get_workflow(dict_configs, cdump='gdas'): # fcst deps = [] - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/INPUT/sfc_data.tile6.nc' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) From abdc1334e3c58d40977b20fb7acabd4c14a80cf6 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 3 Feb 2021 19:28:44 +0000 Subject: [PATCH 03/40] Further init updates for free-forecast mode - move COMPONENT setting from getic to init job - cleanup how getic job sets tarball paths and pulls pgbanl files - move second step of pgbanl pull to init job for consistency - add MODE variable to config.base and its definition to setup scripts - add MODE setting to config.base if-block that adjusts IAU variables - add missing COMPONENT subfolder to data dependencies for getic and init jobs in setup_workflow_fcstonly.py - remove unneeded commented out wavestat job from setup_workflow_fcstonly.py Refs: #178 --- jobs/rocoto/getic.sh | 22 +++++----------------- jobs/rocoto/init.sh | 15 +++++++++++++++ parm/config/config.base.emc.dyn | 5 ++++- ush/rocoto/setup_expt.py | 1 + ush/rocoto/setup_expt_fcstonly.py | 1 + ush/rocoto/setup_workflow_fcstonly.py | 18 ++++-------------- 6 files changed, 30 insertions(+), 32 deletions(-) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index fd07a9ead4f..b3cb741b33e 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -45,8 +45,6 @@ EXTRACT_DIR=${PTMP}/gdas.init_${CDATE}/input OUTDIR=${PTMP}/gdas.init_${CDATE}/output PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory/rh${yy}/${yy}${mm}/${yy}${mm}${dd} -COMPONENT="atmos" - gfs_ver=v16 GETICSH=${GDASINIT_DIR}/get_v16.data.sh @@ -64,9 +62,11 @@ elif [ $yy$mm$dd$hh -lt 2017072000 ]; then elif [ $yy$mm$dd$hh -lt 2019061200 ]; then gfs_ver=v14 GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh + tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.pgrb2_${grid}.tar elif [ $yy$mm$dd$hh -lt 2021020300 ]; then gfs_ver=v15 GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh + tarball=com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar fi export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF @@ -78,25 +78,13 @@ sh ${GETICSH} ${CDUMP} status=$? [[ $status -ne 0 ]] && exit $status -# Copy pgbanl file to ROTDIR for verification/archival - v14+ -cd $EXTRACT_DIR -OUTDIR2=${ROTDIR}/gfs.${yy}${mm}${dd}/${hh}/${COMPONENT} -if [ ! -d ${OUTDIR2} ]; then mkdir -p ${OUTDIR2} ; fi -if [ $gfs_ver = v14 ]; then - for grid in 0p25 0p50 1p00 - do - tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.pgrb2_${grid}.tar - file=gfs.t${hh}z.pgrb2.${grid}.anl - htar -xvf ${PRODHPSSDIR}/${tarball} ./gfs.${yy}${mm}${dd}/${hh}/${file} - mv ${EXTRACT_DIR}/gfs.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR2}/${file} - done -elif [ $gfs_ver = v15 ]; then - tarball=com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar +# Pull pgbanl file for verification/archival - v14+ +if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then + cd $EXTRACT_DIR for grid in 0p25 0p50 1p00 do file=gfs.t${hh}z.pgrb2.${grid}.anl htar -xvf ${PRODHPSSDIR}/${tarball} ./gfs.${yy}${mm}${dd}/${hh}/${file} - mv ${EXTRACT_DIR}/gfs.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR2}/${file} done fi diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh index c36addb2ed5..330846290ee 100755 --- a/jobs/rocoto/init.sh +++ b/jobs/rocoto/init.sh @@ -45,6 +45,8 @@ EXTRACT_DIR=${PTMP}/gdas.init_${CDATE}/input WORKDIR=${PTMP}/gdas.init_${CDATE}/output OUTDIR=${ROTDIR} +COMPONENT="atmos" + gfs_ver=v16 RUNICSH=${GDASINIT_DIR}/run_v16.chgres.sh @@ -76,6 +78,19 @@ sh ${RUNICSH} ${CDUMP} status=$? [[ $status -ne 0 ]] && exit $status +# Copy pgbanl file to ROTDIR for verification/archival - v14+ +if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then + cd $EXTRACT_DIR + for grid in 0p25 0p50 1p00 + do + file=gfs.t${hh}z.pgrb2.${grid}.anl + mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file} + done +fi + +# Clean up EXTRACT_DIR +rm -rf $EXTRACT_DIR + ############################################################### # Exit out cleanly exit 0 diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index 37669926eed..dd3459ce5d3 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -58,6 +58,8 @@ export WAFSF="NO" # WAFS products # use RUNMOS flag (currently in config.vrfy) export REALTIME="YES" +# Experiment mode (cycled or free-forecast) +export MODE="@MODE@" # cycled/free #################################################### # DO NOT ADD MACHINE DEPENDENT STUFF BELOW THIS LINE @@ -209,7 +211,8 @@ export IAU_OFFSET=6 export DOIAU_ENKF=${DOIAU:-"YES"} # Enable 4DIAU for EnKF ensemble export IAUFHRS_ENKF="3,6,9" export IAU_DELTHRS_ENKF=6 -if [[ "$SDATE" = "$CDATE" && $EXP_WARM_START = ".false." ]] || [[ "$DOIAU" = "NO" ]] ; then # Cold starting or IAU off +# Check if cycle is cold starting, DOIAU off, or free-forecast mode +if [[ "$MODE" = "cycled" && "$SDATE" = "$CDATE" && $EXP_WARM_START = ".false." ]] || [[ "$DOIAU" = "NO" ]] || [[ "$MODE" = "free" && $EXP_WARM_START = ".false." ]] ; then export IAU_OFFSET=0 export IAU_FHROT=0 fi diff --git a/ush/rocoto/setup_expt.py b/ush/rocoto/setup_expt.py index afd3b028304..903b91eaf57 100755 --- a/ush/rocoto/setup_expt.py +++ b/ush/rocoto/setup_expt.py @@ -105,6 +105,7 @@ def edit_baseconfig(): .replace('@QUEUE_SERVICE@', queue_service) \ .replace('@PARTITION_BATCH@', partition_batch) \ .replace('@EXP_WARM_START@', exp_warm_start) \ + .replace('@MODE@', 'cycled') \ .replace('@CHGRP_RSTPROD@', chgrp_rstprod) \ .replace('@CHGRP_CMD@', chgrp_cmd) \ .replace('@HPSSARCH@', hpssarch) \ diff --git a/ush/rocoto/setup_expt_fcstonly.py b/ush/rocoto/setup_expt_fcstonly.py index 8a7ffb58dcf..c95d88cea0e 100755 --- a/ush/rocoto/setup_expt_fcstonly.py +++ b/ush/rocoto/setup_expt_fcstonly.py @@ -79,6 +79,7 @@ def edit_baseconfig(): .replace('@QUEUE_SERVICE@', queue_service) \ .replace('@PARTITION_BATCH@', partition_batch) \ .replace('@EXP_WARM_START@', exp_warm_start) \ + .replace('@MODE@', 'free') \ .replace('@CHGRP_RSTPROD@', chgrp_rstprod) \ .replace('@CHGRP_CMD@', chgrp_cmd) \ .replace('@HPSSARCH@', hpssarch) \ diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index 1bd02a8504a..7e6a7779afb 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -245,10 +245,10 @@ def get_workflow(dict_configs, cdump='gdas'): # getic deps = [] - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/INPUT/sfc_data.tile6.nc' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) @@ -259,10 +259,10 @@ def get_workflow(dict_configs, cdump='gdas'): # init - chgres_cube deps = [] - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/INPUT/sfc_data.tile6.nc' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) @@ -380,16 +380,6 @@ def get_workflow(dict_configs, cdump='gdas'): tasks.append(task) tasks.append('\n') - # wavestat - #if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - # deps = [] - # dep_dict = {'type':'task', 'name':'%swavepost' % cdump} - # deps.append(rocoto.add_dependency(dep_dict)) - # dependencies = rocoto.create_dependency(dep=deps) - # task = wfu.create_wf_task('wavestat', cdump=cdump, envar=envars, dependency=dependencies) - # tasks.append(task) - # tasks.append('\n') - # wavegempak if do_wave in ['Y', 'YES'] and do_gempak in ['Y', 'YES']: deps = [] From ed74d83b64cbc98a36698dfb39b98fc860892ea4 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 4 Feb 2021 13:28:59 -0600 Subject: [PATCH 04/40] Adjustments for Orion, dependencies, and using DATA folder - update getic and init jobs to use DATA folder and ROTDIR - move pgb copy back to getic job since it now dumps into ROTDIR - update setup_workflow_fcstonly.py to turn off getic on Orion and adjust dependencies for init for input files from supported versions Refs: #178 --- jobs/rocoto/getic.sh | 10 ++++-- jobs/rocoto/init.sh | 25 ++++++-------- ush/rocoto/setup_workflow_fcstonly.py | 47 +++++++++++++++------------ 3 files changed, 44 insertions(+), 38 deletions(-) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index b3cb741b33e..f5ff1c96980 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -41,8 +41,11 @@ mm=$(echo $CDATE | cut -c5-6) dd=$(echo $CDATE | cut -c7-8) hh=${cyc:-$(echo $CDATE | cut -c9-10)} -EXTRACT_DIR=${PTMP}/gdas.init_${CDATE}/input -OUTDIR=${PTMP}/gdas.init_${CDATE}/output +export DATA=${DATA:-${DATAROOT}/init} + +EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR} +WORKDIR=${WORKDIR:-$DATA} +OUTDIR=${OUTDIR:-$ROTDIR} PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory/rh${yy}/${yy}${mm}/${yy}${mm}${dd} gfs_ver=v16 @@ -84,7 +87,8 @@ if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then for grid in 0p25 0p50 1p00 do file=gfs.t${hh}z.pgrb2.${grid}.anl - htar -xvf ${PRODHPSSDIR}/${tarball} ./gfs.${yy}${mm}${dd}/${hh}/${file} + htar -xvf ${PRODHPSSDIR}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file} + mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file} done fi diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh index 330846290ee..5b4ee7dedd6 100755 --- a/jobs/rocoto/init.sh +++ b/jobs/rocoto/init.sh @@ -41,9 +41,11 @@ mm=$(echo $CDATE | cut -c5-6) dd=$(echo $CDATE | cut -c7-8) hh=${cyc:-$(echo $CDATE | cut -c9-10)} -EXTRACT_DIR=${PTMP}/gdas.init_${CDATE}/input -WORKDIR=${PTMP}/gdas.init_${CDATE}/output -OUTDIR=${ROTDIR} +export DATA=${DATA:-${DATAROOT}/init} + +EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR} +WORKDIR=${WORKDIR:-$DATA} +OUTDIR=${OUTDIR:-$ROTDIR} COMPONENT="atmos" @@ -78,18 +80,11 @@ sh ${RUNICSH} ${CDUMP} status=$? [[ $status -ne 0 ]] && exit $status -# Copy pgbanl file to ROTDIR for verification/archival - v14+ -if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then - cd $EXTRACT_DIR - for grid in 0p25 0p50 1p00 - do - file=gfs.t${hh}z.pgrb2.${grid}.anl - mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file} - done -fi - -# Clean up EXTRACT_DIR -rm -rf $EXTRACT_DIR +########################################## +# Remove the Temporary working directory +########################################## +cd $DATAROOT +[[ $KEEPDATA = "NO" ]] && rm -rf $DATA ############################################################### # Exit out cleanly diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index 7e6a7779afb..9a1ff4e6287 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -234,6 +234,7 @@ def get_workflow(dict_configs, cdump='gdas'): envars.append(rocoto.create_envar(name='cyc', value='@H')) base = dict_configs['base'] + machine = base.get('machine', wfu.detectMachine()) do_wave = base.get('DO_WAVE', 'NO').upper() do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() do_gempak = base.get('DO_GEMPAK', 'NO').upper() @@ -244,38 +245,44 @@ def get_workflow(dict_configs, cdump='gdas'): tasks = [] # getic + if machine not in ['ORION']: + deps = [] + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' + dep_dict = {'type':'data', 'data':data} + deps.append(rocoto.add_dependency(dep_dict)) + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' + dep_dict = {'type':'data', 'data':data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) + + task = wfu.create_wf_task('getic', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # init deps = [] - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/gfs.t@Hz.sanl' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) - - task = wfu.create_wf_task('getic', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # init - chgres_cube - deps = [] - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/gfs.t@Hz.atmanl.nemsio' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - deps = [] - dep_dict = {'type': 'task', 'name': '%sgetic' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep=deps) + if machine not in ['ORION']: + deps = [] + dep_dict = {'type': 'task', 'name': '%sgetic' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies2 = rocoto.create_dependency(dep=deps) deps = [] deps.append(dependencies) - deps.append(dependencies2) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + if machine not in ['ORION']: + deps.append(dependencies2) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('init', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) From 7a34968e3a66fc31127de1450d5ce194d9b9681b Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 4 Feb 2021 14:32:30 -0600 Subject: [PATCH 05/40] Small adjustments from testing - add needed exports in getic.sh - add check to init for whether it needs to run - add check to init for copying pgbanl files when getic doesn't run - remove atmos subfolder from init dependencies for older versions Refs: #178 --- jobs/rocoto/getic.sh | 6 ++--- jobs/rocoto/init.sh | 33 ++++++++++++++++++++------- ush/rocoto/setup_workflow_fcstonly.py | 4 ++-- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index f5ff1c96980..c6798008922 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -43,9 +43,9 @@ hh=${cyc:-$(echo $CDATE | cut -c9-10)} export DATA=${DATA:-${DATAROOT}/init} -EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR} -WORKDIR=${WORKDIR:-$DATA} -OUTDIR=${OUTDIR:-$ROTDIR} +export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR} +export WORKDIR=${WORKDIR:-$DATA} +export OUTDIR=${OUTDIR:-$ROTDIR} PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory/rh${yy}/${yy}${mm}/${yy}${mm}${dd} gfs_ver=v16 diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh index 5b4ee7dedd6..8a1078f2fbf 100755 --- a/jobs/rocoto/init.sh +++ b/jobs/rocoto/init.sh @@ -43,9 +43,9 @@ hh=${cyc:-$(echo $CDATE | cut -c9-10)} export DATA=${DATA:-${DATAROOT}/init} -EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR} -WORKDIR=${WORKDIR:-$DATA} -OUTDIR=${OUTDIR:-$ROTDIR} +export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR} +export WORKDIR=${WORKDIR:-$DATA} +export OUTDIR=${OUTDIR:-$ROTDIR} COMPONENT="atmos" @@ -74,11 +74,28 @@ fi export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF export LEVS gfs_ver -# Run chgres_cube -if [ ! -d $OUTDIR ]; then mkdir -p $OUTDIR ; fi -sh ${RUNICSH} ${CDUMP} -status=$? -[[ $status -ne 0 ]] && exit $status +# Check if init is needed and run if so +if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then + echo "Detected v16 C768 warm starts, will not run init. Exiting..." +else + # Run chgres_cube + if [ ! -d $OUTDIR ]; then mkdir -p $OUTDIR ; fi + sh ${RUNICSH} ${CDUMP} + status=$? + [[ $status -ne 0 ]] && exit $status +fi + +# Check for pgbanl files and move +if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then + cd $EXTRACT_DIR + for grid in 0p25 0p50 1p00 + do + file=gfs.t${hh}z.pgrb2.${grid}.anl + if [ -f ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ]; then + mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file} + fi + done +fi ########################################## # Remove the Temporary working directory diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index 9a1ff4e6287..f1f27c672e0 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -261,10 +261,10 @@ def get_workflow(dict_configs, cdump='gdas'): # init deps = [] - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/gfs.t@Hz.sanl' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.sanl' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/gfs.t@Hz.atmanl.nemsio' + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nemsio' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' From 613a181c040682dfb41997c64c2d220a33eeecbe Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 4 Feb 2021 20:57:07 +0000 Subject: [PATCH 06/40] Adjusting from testing - add COMPONENT setting back into getic.sh - add gfsinit as dependency to gfswaveprep since UFS_UTILS gdas_init scripts remove the atmos folder Refs: #178 --- jobs/rocoto/getic.sh | 2 ++ ush/rocoto/setup_workflow_fcstonly.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index c6798008922..0f3557346a1 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -48,6 +48,8 @@ export WORKDIR=${WORKDIR:-$DATA} export OUTDIR=${OUTDIR:-$ROTDIR} PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory/rh${yy}/${yy}${mm}/${yy}${mm}${dd} +COMPONENT="atmos" + gfs_ver=v16 GETICSH=${GDASINIT_DIR}/get_v16.data.sh diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index f1f27c672e0..49c004e917c 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -299,7 +299,9 @@ def get_workflow(dict_configs, cdump='gdas'): deps = [] dep_dict = {'type': 'task', 'name': '%swaveinit' % cdump} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + dep_dict = {'type': 'task', 'name': '%sinit' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('waveprep', cdump=cdump, envar=envars, dependency=dependencies) tasks.append(task) tasks.append('\n') From 77d7408b759a761d1697ad672dd4feae14d9c5b1 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 9 Feb 2021 18:49:05 +0000 Subject: [PATCH 07/40] Move GFSv16 version date and set C96 to use 4 threads - change config.fv3 to use 4 threads for C96 (too few nodes with 1 thread) - update GFSv16 version if-block date in getic and init scripts to use updated/tentative implementation date; update when finalized Refs: #178 --- jobs/rocoto/getic.sh | 2 +- jobs/rocoto/init.sh | 2 +- parm/config/config.fv3 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index 0f3557346a1..02e5e1faf22 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -68,7 +68,7 @@ elif [ $yy$mm$dd$hh -lt 2019061200 ]; then gfs_ver=v14 GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.pgrb2_${grid}.tar -elif [ $yy$mm$dd$hh -lt 2021020300 ]; then +elif [ $yy$mm$dd$hh -lt 2021031700 ]; then gfs_ver=v15 GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh tarball=com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh index 8a1078f2fbf..a385557ae7e 100755 --- a/jobs/rocoto/init.sh +++ b/jobs/rocoto/init.sh @@ -66,7 +66,7 @@ elif [ $yy$mm$dd$hh -lt 2017072000 ]; then elif [ $yy$mm$dd$hh -lt 2019061200 ]; then gfs_ver=v14 RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh -elif [ $yy$mm$dd$hh -lt 2021020300 ]; then +elif [ $yy$mm$dd$hh -lt 2021031700 ]; then gfs_ver=v15 RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh fi diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 index 3fdfc468a2f..5fbe168916d 100755 --- a/parm/config/config.fv3 +++ b/parm/config/config.fv3 @@ -59,7 +59,7 @@ case $case_in in export layout_y_gfs=4 export npe_wav=140 export npe_wav_gfs=140 - export nth_fv3=1 + export nth_fv3=4 export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 export WRTTASK_PER_GROUP=$npe_node_max From 10ed5f1152c385a4d19864ac5924b77dd66dc75e Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 9 Feb 2021 19:08:38 +0000 Subject: [PATCH 08/40] Add wave jobs to arch job dependencies Refs: #178 --- ush/rocoto/setup_workflow_fcstonly.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index 49c004e917c..589ee4a569b 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -627,6 +627,13 @@ def get_workflow(dict_configs, cdump='gdas'): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type':'streq', 'left':'&ARCHIVE_TO_HPSS;', 'right':'YES'} deps.append(rocoto.add_dependency(dep_dict)) + if do_wave in ['Y', 'YES']: + dep_dict = {'type': 'task', 'name': '%swavepostsbs' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': '%swavepostpnt' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': '%swavepostbndpnt' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies, final=True) tasks.append(task) From 181606755de789d1764d1f21a5a2e72ea302da34 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 12 Feb 2021 14:19:51 +0000 Subject: [PATCH 09/40] Updates for archival, NSST, and v16 retros - add variables: MODE to dilineate free-forecast and cycled modes; RETRO variable to toggle between v16 ops and retro inputs - update arch step to archive INPUT files instead of RESTART files when in free (free-forecast) mode; other small fixes to arch step - set NSST to spinup before availability date - added support for running off v16 retrospective inputs - fix missing bufrsnd job in free-forecast mode - update resource settings for C48 and C96; further refinement coming Refs: #178 --- jobs/rocoto/arch.sh | 8 +++- jobs/rocoto/getic.sh | 47 +++++++++++-------- jobs/rocoto/init.sh | 41 ++++++++++------- parm/config/config.base.emc.dyn | 1 + parm/config/config.fv3 | 30 ++++++------ parm/config/config.getic | 2 + parm/config/config.init | 2 +- parm/config/config.nsst | 6 +-- ush/hpssarch_gen.sh | 66 +++++++++++++++++++-------- ush/rocoto/setup_workflow_fcstonly.py | 45 +++++++++++++++++- 10 files changed, 170 insertions(+), 78 deletions(-) diff --git a/jobs/rocoto/arch.sh b/jobs/rocoto/arch.sh index ee3af795c26..85a9e6406aa 100755 --- a/jobs/rocoto/arch.sh +++ b/jobs/rocoto/arch.sh @@ -181,7 +181,13 @@ if [ $CDUMP = "gfs" ]; then #for targrp in gfs_flux gfs_netcdf/nemsio gfs_pgrb2b; do if [ ${SAVEFCSTNEMSIO:-"YES"} = "YES" ]; then - for targrp in gfs_flux gfs_${format}a gfs_${format}b gfs_pgrb2b; do + if [ $MODE = "cycled" ]; then + targrp_list=`gfs_flux gfs_${format}a gfs_${format}b gfs_pgrb2b` + elif [ $MODE = "free" ]; then + targrp_list=`gfs_flux gfs_${format}b gfs_pgrb2b` + fi + #for targrp in gfs_flux gfs_${format}a gfs_${format}b gfs_pgrb2b; do + for targrp in $targrp_list; do htar -P -cvf $ATARDIR/$CDATE/${targrp}.tar `cat $ARCH_LIST/${targrp}.txt` status=$? if [ $status -ne 0 -a $CDATE -ge $firstday ]; then diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index 02e5e1faf22..8a8dfee7004 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -20,7 +20,7 @@ status=$? ############################################################### # Source relevant configs -configs="base getic" +configs="base init" for config in $configs; do . $EXPDIR/config.${config} status=$? @@ -53,25 +53,32 @@ COMPONENT="atmos" gfs_ver=v16 GETICSH=${GDASINIT_DIR}/get_v16.data.sh -# No ENKF data prior to 2012/05/21/00z -if [ $yy$mm$dd$hh -lt 2012052100 ]; then - set +x - echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA - exit 2 -elif [ $yy$mm$dd$hh -lt 2016051000 ]; then - gfs_ver=v12 - GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh -elif [ $yy$mm$dd$hh -lt 2017072000 ]; then - gfs_ver=v13 - GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh -elif [ $yy$mm$dd$hh -lt 2019061200 ]; then - gfs_ver=v14 - GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh - tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.pgrb2_${grid}.tar -elif [ $yy$mm$dd$hh -lt 2021031700 ]; then - gfs_ver=v15 - GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh - tarball=com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar +RETRO=${RETRO:-"NO"} +if [ $RETRO = "YES" ]; then + GETICSH=${GDASINIT_DIR}/get_v16retro.data.sh +fi + +if [ $RETRO = "NO" ]; then # Operational input + # No ENKF data prior to 2012/05/21/00z + if [ $yy$mm$dd$hh -lt 2012052100 ]; then + set +x + echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA + exit 2 + elif [ $yy$mm$dd$hh -lt 2016051000 ]; then + gfs_ver=v12 + GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh + elif [ $yy$mm$dd$hh -lt 2017072000 ]; then + gfs_ver=v13 + GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh + elif [ $yy$mm$dd$hh -lt 2019061200 ]; then + gfs_ver=v14 + GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh + tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.pgrb2_${grid}.tar + elif [ $yy$mm$dd$hh -lt 2021031700 ]; then + gfs_ver=v15 + GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh + tarball=com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar + fi fi export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh index a385557ae7e..b944216a838 100755 --- a/jobs/rocoto/init.sh +++ b/jobs/rocoto/init.sh @@ -52,23 +52,30 @@ COMPONENT="atmos" gfs_ver=v16 RUNICSH=${GDASINIT_DIR}/run_v16.chgres.sh -# No ENKF data prior to 2012/05/21/00z -if [ $yy$mm$dd$hh -lt 2012052100 ]; then - set +x - echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA - exit 2 -elif [ $yy$mm$dd$hh -lt 2016051000 ]; then - gfs_ver=v12 - RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh -elif [ $yy$mm$dd$hh -lt 2017072000 ]; then - gfs_ver=v13 - RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh -elif [ $yy$mm$dd$hh -lt 2019061200 ]; then - gfs_ver=v14 - RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh -elif [ $yy$mm$dd$hh -lt 2021031700 ]; then - gfs_ver=v15 - RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh +RETRO=${RETRO:-"NO"} +if [ $RETRO = "YES" ]; then + GETICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh +fi + +if [ $RETRO = "NO" ]; then # Operational input + # No ENKF data prior to 2012/05/21/00z + if [ $yy$mm$dd$hh -lt 2012052100 ]; then + set +x + echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA + exit 2 + elif [ $yy$mm$dd$hh -lt 2016051000 ]; then + gfs_ver=v12 + RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh + elif [ $yy$mm$dd$hh -lt 2017072000 ]; then + gfs_ver=v13 + RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh + elif [ $yy$mm$dd$hh -lt 2019061200 ]; then + gfs_ver=v14 + RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh + elif [ $yy$mm$dd$hh -lt 2021031700 ]; then + gfs_ver=v15 + RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh + fi fi export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index dd3459ce5d3..5634905a2f7 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -226,6 +226,7 @@ export gldas_cyc=00 # run wave component export DO_WAVE="YES" +if [[ "$SDATE" -lt "2019020100" ]]; then DO_WAVE="NO" ; fi # no rtofs in GDA export WAVE_CDUMP="both" # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 index 5fbe168916d..410a5ed4a7d 100755 --- a/parm/config/config.fv3 +++ b/parm/config/config.fv3 @@ -36,33 +36,33 @@ fi # (Standard) Model resolution dependent variables case $case_in in "C48") - export DELTIM=450 - export layout_x=2 - export layout_y=4 - export layout_x_gfs=2 - export layout_y_gfs=4 - export npe_wav=140 - export npe_wav_gfs=140 + export DELTIM=1200 + export layout_x=3 + export layout_y=2 + export layout_x_gfs=3 + export layout_y_gfs=2 + export npe_wav=16 + export npe_wav_gfs=16 export nth_fv3=1 export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 export WRTTASK_PER_GROUP=$npe_node_max export WRITE_GROUP_GFS=1 export WRTTASK_PER_GROUP_GFS=$npe_node_max - export WRTIOBUF="4M" + export WRTIOBUF="1M" ;; "C96") - export DELTIM=450 - export layout_x=4 + export DELTIM=900 + export layout_x=6 export layout_y=4 - export layout_x_gfs=4 + export layout_x_gfs=6 export layout_y_gfs=4 - export npe_wav=140 - export npe_wav_gfs=140 - export nth_fv3=4 + export npe_wav=40 + export npe_wav_gfs=40 + export nth_fv3=1 export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 - export WRTTASK_PER_GROUP=$npe_node_max + export WRTTASK_PER_GROUP=6 export WRITE_GROUP_GFS=1 export WRTTASK_PER_GROUP_GFS=$npe_node_max export WRTIOBUF="4M" diff --git a/parm/config/config.getic b/parm/config/config.getic index 64f7159cc98..52498a01d3d 100755 --- a/parm/config/config.getic +++ b/parm/config/config.getic @@ -13,4 +13,6 @@ GDASINIT_DIR=${UFS_DIR}/util/gdas_init EXTRACT_DATA="yes" +RETRO="NO" # YES = Pull v16 inputs from retrospective parallels; NO = use operational inputs + echo "END: config.getic" diff --git a/parm/config/config.init b/parm/config/config.init index f9efbde888b..a473d9747a5 100755 --- a/parm/config/config.init +++ b/parm/config/config.init @@ -11,7 +11,7 @@ echo "BEGIN: config.init" UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd GDASINIT_DIR=${UFS_DIR}/util/gdas_init -RUN_CHGRES="yes" +RETRO="NO" # YES = Pull v16 inputs from retrospective parallels; NO = use operational inputs CRES_HIRES=$CASE CRES_ENKF=$CASE_ENKF diff --git a/parm/config/config.nsst b/parm/config/config.nsst index a59ca19495f..ef8767834d1 100755 --- a/parm/config/config.nsst +++ b/parm/config/config.nsst @@ -12,9 +12,9 @@ export NST_MODEL=2 # nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON, export NST_SPINUP=0 -#if [[ "$CDATE" = $SDATE ]]; then -# export NST_SPINUP=1 -#fi +if [[ "$CDATE" -lt "2017072000" ]]; then + export NST_SPINUP=1 +fi # nstf_name(3) : NST_RESV (Reserved, NSST Analysis) : 0 = OFF, 1 = ON export NST_RESV=0 diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh index 1f4c8ed542b..4e65d7dbdf7 100755 --- a/ush/hpssarch_gen.sh +++ b/ush/hpssarch_gen.sh @@ -41,17 +41,23 @@ if [ $type = "gfs" ]; then rm -f gfsb.txt rm -f gfs_pgrb2b.txt rm -f gfs_flux.txt - rm -f gfs_${format}a.txt - rm -f gfs_${format}b.txt rm -f gfs_restarta.txt touch gfsa.txt touch gfsb.txt touch gfs_pgrb2b.txt touch gfs_flux.txt - touch gfs_${format}a.txt - touch gfs_${format}b.txt touch gfs_restarta.txt + if [ $MODE = "cycled" ]; then + rm -f gfs_${format}a.txt + touch gfs_${format}a.txt + fi + + if [ $OUTPUT_HISTORY = ".true." ]; then + rm -f gfs_${format}b.txt + touch gfs_${format}b.txt + fi + if [ $DO_DOWN = "YES" ]; then rm -f gfs_downstream.txt touch gfs_downstream.txt @@ -69,11 +75,13 @@ if [ $type = "gfs" ]; then echo "${dirname}${head}pgrb2b.0p50.anl.idx " >>gfs_pgrb2b.txt echo "./logs/${CDATE}/gfs*.log " >>gfsa.txt - echo "${dirname}${head}gsistat " >>gfsa.txt - echo "${dirname}${head}nsstbufr " >>gfsa.txt - echo "${dirname}${head}prepbufr " >>gfsa.txt - echo "${dirname}${head}prepbufr_pre-qc " >>gfsa.txt - echo "${dirname}${head}prepbufr.acft_profiles " >>gfsa.txt + if [ $MODE = "cycled" ]; then + echo "${dirname}${head}gsistat " >>gfsa.txt + echo "${dirname}${head}nsstbufr " >>gfsa.txt + echo "${dirname}${head}prepbufr " >>gfsa.txt + echo "${dirname}${head}prepbufr_pre-qc " >>gfsa.txt + echo "${dirname}${head}prepbufr.acft_profiles " >>gfsa.txt + fi echo "${dirname}${head}pgrb2.0p25.anl " >>gfsa.txt echo "${dirname}${head}pgrb2.0p25.anl.idx " >>gfsa.txt echo "${dirname}avno.t${cyc}z.cyclone.trackatcfunix " >>gfsa.txt @@ -145,11 +153,13 @@ if [ $type = "gfs" ]; then #.................. - echo "${dirname}${head}atmanl${SUFFIX} " >>gfs_${format}a.txt - echo "${dirname}${head}sfcanl${SUFFIX} " >>gfs_${format}a.txt - echo "${dirname}${head}atmi*.nc " >>gfs_${format}a.txt - echo "${dirname}${head}dtfanl.nc " >>gfs_${format}a.txt - echo "${dirname}${head}loginc.txt " >>gfs_${format}a.txt + if [ $MODE = "cycled" ]; then + echo "${dirname}${head}atmanl${SUFFIX} " >>gfs_${format}a.txt + echo "${dirname}${head}sfcanl${SUFFIX} " >>gfs_${format}a.txt + echo "${dirname}${head}atmi*.nc " >>gfs_${format}a.txt + echo "${dirname}${head}dtfanl.nc " >>gfs_${format}a.txt + echo "${dirname}${head}loginc.txt " >>gfs_${format}a.txt + fi #.................. if [ $OUTPUT_HISTORY = ".true." ]; then @@ -163,12 +173,28 @@ if [ $type = "gfs" ]; then fi #.................. - echo "${dirname}RESTART/*0000.sfcanl_data.tile1.nc " >>gfs_restarta.txt - echo "${dirname}RESTART/*0000.sfcanl_data.tile2.nc " >>gfs_restarta.txt - echo "${dirname}RESTART/*0000.sfcanl_data.tile3.nc " >>gfs_restarta.txt - echo "${dirname}RESTART/*0000.sfcanl_data.tile4.nc " >>gfs_restarta.txt - echo "${dirname}RESTART/*0000.sfcanl_data.tile5.nc " >>gfs_restarta.txt - echo "${dirname}RESTART/*0000.sfcanl_data.tile6.nc " >>gfs_restarta.txt + if [ $MODE = "cycled" ]; then + echo "${dirname}RESTART/*0000.sfcanl_data.tile1.nc " >>gfs_restarta.txt + echo "${dirname}RESTART/*0000.sfcanl_data.tile2.nc " >>gfs_restarta.txt + echo "${dirname}RESTART/*0000.sfcanl_data.tile3.nc " >>gfs_restarta.txt + echo "${dirname}RESTART/*0000.sfcanl_data.tile4.nc " >>gfs_restarta.txt + echo "${dirname}RESTART/*0000.sfcanl_data.tile5.nc " >>gfs_restarta.txt + echo "${dirname}RESTART/*0000.sfcanl_data.tile6.nc " >>gfs_restarta.txt + elif [ $MODE = "free" ]; then + echo "${dirname}INPUT/gfs_ctrl.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/gfs_data.tile1.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/gfs_data.tile2.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/gfs_data.tile3.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/gfs_data.tile4.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/gfs_data.tile5.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/gfs_data.tile6.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/sfc_data.tile1.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/sfc_data.tile2.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/sfc_data.tile3.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/sfc_data.tile4.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/sfc_data.tile5.nc " >>gfs_restarta.txt + echo "${dirname}INPUT/sfc_data.tile6.nc " >>gfs_restarta.txt + fi #.................. if [ $DO_WAVE = "YES" ]; then diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index 589ee4a569b..767984eb079 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -27,7 +27,7 @@ import rocoto import workflow_utils as wfu -taskplan = ['getic', 'init', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'vrfy', 'metp', 'arch'] +taskplan = ['getic', 'init', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'postsnd', 'gempak', 'awips', 'vrfy', 'metp', 'arch'] def main(): parser = ArgumentParser(description='Setup XML workflow and CRONTAB for a forecast only experiment.', formatter_class=ArgumentDefaultsHelpFormatter) @@ -158,6 +158,7 @@ def get_resources(dict_configs, cdump='gdas'): scheduler = wfu.get_scheduler(machine) do_wave = base.get('DO_WAVE', 'NO').upper() + do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() do_gempak = base.get('DO_GEMPAK', 'NO').upper() do_awips = base.get('DO_AWIPS', 'NO').upper() do_metp = base.get('DO_METP', 'NO').upper() @@ -237,6 +238,7 @@ def get_workflow(dict_configs, cdump='gdas'): machine = base.get('machine', wfu.detectMachine()) do_wave = base.get('DO_WAVE', 'NO').upper() do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() + do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() do_gempak = base.get('DO_GEMPAK', 'NO').upper() do_awips = base.get('DO_AWIPS', 'NO').upper() do_wafs = base.get('WAFSF', 'NO').upper() @@ -267,6 +269,9 @@ def get_workflow(dict_configs, cdump='gdas'): data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nemsio' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nc' + dep_dict = {'type':'data', 'data':data} + deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) @@ -593,6 +598,44 @@ def get_workflow(dict_configs, cdump='gdas'): tasks.append(task) tasks.append('\n') + #postsnd + if do_bufrsnd in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'task', 'name': '%sfcst' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('postsnd', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # awips + if do_awips in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'metatask', 'name': '%spost' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') + fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') + ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') + awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'], cdump=cdump) + vardict = {varname2: varval2, varname3: varval3} + task = wfu.create_wf_task('awips', cdump=cdump, envar=awipsenvars, dependency=dependencies, + metatask='awips', varname=varname1, varval=varval1, vardict=vardict) + tasks.append(task) + tasks.append('\n') + + # gempak + if do_gempak in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'metatask', 'name': '%spost' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('gempak', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + # vrfy deps = [] dep_dict = {'type':'metatask', 'name':'%spost' % cdump} From d32199cd58329b4b2c9c5c3e84e7d8d0801305e0 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 4 Mar 2021 17:43:58 +0000 Subject: [PATCH 10/40] Move getic/init settings from scripts to configs - consolidate RETRO variable in config.getic - move configuration settings from getic/init scripts to configs - add dependency to setup_workflow_fcstonly.py - add getic and init to task list at top of config.resources Refs: #178 --- jobs/rocoto/getic.sh | 54 +++++---------------------- jobs/rocoto/init.sh | 2 +- parm/config/config.getic | 36 ++++++++++++++++-- parm/config/config.init | 35 ++++++++++++++--- parm/config/config.resources | 1 + ush/rocoto/setup_workflow_fcstonly.py | 3 ++ 6 files changed, 77 insertions(+), 54 deletions(-) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index 8a8dfee7004..6523acce5cf 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -20,7 +20,7 @@ status=$? ############################################################### # Source relevant configs -configs="base init" +configs="base getic init" for config in $configs; do . $EXPDIR/config.${config} status=$? @@ -36,53 +36,19 @@ status=$? ############################################################### # Set script and dependency variables -yy=$(echo $CDATE | cut -c1-4) -mm=$(echo $CDATE | cut -c5-6) -dd=$(echo $CDATE | cut -c7-8) -hh=${cyc:-$(echo $CDATE | cut -c9-10)} +export yy=$(echo $CDATE | cut -c1-4) +export mm=$(echo $CDATE | cut -c5-6) +export dd=$(echo $CDATE | cut -c7-8) +export hh=${cyc:-$(echo $CDATE | cut -c9-10)} export DATA=${DATA:-${DATAROOT}/init} - export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR} export WORKDIR=${WORKDIR:-$DATA} export OUTDIR=${OUTDIR:-$ROTDIR} -PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory/rh${yy}/${yy}${mm}/${yy}${mm}${dd} - -COMPONENT="atmos" - -gfs_ver=v16 -GETICSH=${GDASINIT_DIR}/get_v16.data.sh - -RETRO=${RETRO:-"NO"} -if [ $RETRO = "YES" ]; then - GETICSH=${GDASINIT_DIR}/get_v16retro.data.sh -fi - -if [ $RETRO = "NO" ]; then # Operational input - # No ENKF data prior to 2012/05/21/00z - if [ $yy$mm$dd$hh -lt 2012052100 ]; then - set +x - echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA - exit 2 - elif [ $yy$mm$dd$hh -lt 2016051000 ]; then - gfs_ver=v12 - GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh - elif [ $yy$mm$dd$hh -lt 2017072000 ]; then - gfs_ver=v13 - GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh - elif [ $yy$mm$dd$hh -lt 2019061200 ]; then - gfs_ver=v14 - GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh - tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.pgrb2_${grid}.tar - elif [ $yy$mm$dd$hh -lt 2021031700 ]; then - gfs_ver=v15 - GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh - tarball=com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar - fi -fi - -export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF -export LEVS gfs_ver +export PRODHPSSDIR=${PRODHPSSDIR:-/NCEPPROD/hpssprod/runhistory} +export COMPONENT="atmos" +export gfs_ver=${gfs_ver:-v16} +export GETICSH=${GETICSH:-${GDASINIT_DIR}/get_v16.data.sh} # Run get data script if [ ! -d $EXTRACT_DIR ]; then mkdir -p $EXTRACT_DIR ; fi @@ -96,7 +62,7 @@ if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then for grid in 0p25 0p50 1p00 do file=gfs.t${hh}z.pgrb2.${grid}.anl - htar -xvf ${PRODHPSSDIR}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file} + htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file} mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file} done fi diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh index b944216a838..1671bbb1057 100755 --- a/jobs/rocoto/init.sh +++ b/jobs/rocoto/init.sh @@ -20,7 +20,7 @@ status=$? ############################################################### # Source relevant configs -configs="base init" +configs="base getic init" for config in $configs; do . $EXPDIR/config.${config} status=$? diff --git a/parm/config/config.getic b/parm/config/config.getic index 52498a01d3d..ec396849ae7 100755 --- a/parm/config/config.getic +++ b/parm/config/config.getic @@ -8,11 +8,39 @@ echo "BEGIN: config.getic" # Get task specific resources . $EXPDIR/config.resources getic -UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd -GDASINIT_DIR=${UFS_DIR}/util/gdas_init +export RETRO="NO" # YES = Pull v16 inputs from retrospective parallels; NO = use operational inputs +export gfs_ver="v16" # Default = v16 -EXTRACT_DATA="yes" +export UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd +export GDASINIT_DIR=${UFS_DIR}/util/gdas_init -RETRO="NO" # YES = Pull v16 inputs from retrospective parallels; NO = use operational inputs +export PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory +export GETICSH=${GDASINIT_DIR}/get_v16.data.sh + +if [ ${RETRO:-"NO"} = "YES" ]; then + export GETICSH=${GDASINIT_DIR}/get_v16retro.data.sh +fi + +if [ ${RETRO:-"NO"} = "NO" ]; then # Operational input + # No ENKF data prior to 2012/05/21/00z + if [[ "$CDATE" -lt "2012052100" ]]; then + set +x + echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA + elif [[ "$CDATE" -lt "2016051000" ]]; then + export gfs_ver=v12 + export GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh + elif [[ "$CDATE" -lt "2017072000" ]]; then + export gfs_ver=v13 + export GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh + elif [[ "$CDATE" -lt "2019061200" ]]; then + export gfs_ver=v14 + export GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh + export tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${PDY}_${cyc}.pgrb2_${grid}.tar + elif [[ "$CDATE" -lt "2021031700" ]]; then + export gfs_ver=v15 + export GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh + export tarball=com_gfs_prod_gfs.${PDY}_${cyc}.gfs_pgrb2.tar + fi +fi echo "END: config.getic" diff --git a/parm/config/config.init b/parm/config/config.init index a473d9747a5..212fa422df3 100755 --- a/parm/config/config.init +++ b/parm/config/config.init @@ -8,12 +8,37 @@ echo "BEGIN: config.init" # Get task specific resources . $EXPDIR/config.resources init -UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd -GDASINIT_DIR=${UFS_DIR}/util/gdas_init +# Get task specific resources +. $EXPDIR/config.getic + +export UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd +export GDASINIT_DIR=${UFS_DIR}/util/gdas_init + +export CRES_HIRES=$CASE +export CRES_ENKF=$CASE_ENKF -RETRO="NO" # YES = Pull v16 inputs from retrospective parallels; NO = use operational inputs +if [ ${RETRO:-"NO"} = "YES" ]; then + export GETICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh +fi -CRES_HIRES=$CASE -CRES_ENKF=$CASE_ENKF +if [ ${RETRO:-"NO"} = "NO" ]; then # Operational input + # No ENKF data prior to 2012/05/21/00z + if [[ "$CDATE" -lt "2012052100" ]]; then + set +x + echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA + elif [[ "$CDATE" -lt "2016051000" ]]; then + export gfs_ver=v12 + export RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh + elif [[ "$CDATE" -lt "2017072000" ]]; then + export gfs_ver=v13 + export RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh + elif [[ "$CDATE" -lt "2019061200" ]]; then + export gfs_ver=v14 + export RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh + elif [[ "$CDATE" -lt "2021031700" ]]; then + export gfs_ver=v15 + export RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh + fi +fi echo "END: config.init" diff --git a/parm/config/config.resources b/parm/config/config.resources index d62424aa324..ac081db9768 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -8,6 +8,7 @@ if [ $# -ne 1 ]; then echo "Must specify an input task argument to set resource variables!" echo "argument can be any one of the following:" + echo "getic init" echo "anal analcalc analdiag gldas fcst post vrfy metp arch echgres" echo "eobs ediag eomg eupd ecen esfc efcs epos earc" echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostpnt" diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index 767984eb079..195734e1af0 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -272,6 +272,9 @@ def get_workflow(dict_configs, cdump='gdas'): data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/gfs.t@Hz.atmanl.nc' + dep_dict = {'type':'data', 'data':data} + deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' dep_dict = {'type':'data', 'data':data} deps.append(rocoto.add_dependency(dep_dict)) From be0ed38a53d2422923ccd5380d610785e61deafa Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 4 Mar 2021 18:06:24 +0000 Subject: [PATCH 11/40] Consolidate/cleanup init config and script Refs: #178 --- jobs/rocoto/init.sh | 46 +++++++---------------------------------- parm/config/config.init | 2 +- 2 files changed, 8 insertions(+), 40 deletions(-) diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh index 1671bbb1057..2fb81b268b6 100755 --- a/jobs/rocoto/init.sh +++ b/jobs/rocoto/init.sh @@ -36,50 +36,18 @@ status=$? ############################################################### # Set script and dependency variables -yy=$(echo $CDATE | cut -c1-4) -mm=$(echo $CDATE | cut -c5-6) -dd=$(echo $CDATE | cut -c7-8) -hh=${cyc:-$(echo $CDATE | cut -c9-10)} +export yy=$(echo $CDATE | cut -c1-4) +export mm=$(echo $CDATE | cut -c5-6) +export dd=$(echo $CDATE | cut -c7-8) +export hh=${cyc:-$(echo $CDATE | cut -c9-10)} export DATA=${DATA:-${DATAROOT}/init} - export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR} export WORKDIR=${WORKDIR:-$DATA} export OUTDIR=${OUTDIR:-$ROTDIR} - -COMPONENT="atmos" - -gfs_ver=v16 -RUNICSH=${GDASINIT_DIR}/run_v16.chgres.sh - -RETRO=${RETRO:-"NO"} -if [ $RETRO = "YES" ]; then - GETICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh -fi - -if [ $RETRO = "NO" ]; then # Operational input - # No ENKF data prior to 2012/05/21/00z - if [ $yy$mm$dd$hh -lt 2012052100 ]; then - set +x - echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA - exit 2 - elif [ $yy$mm$dd$hh -lt 2016051000 ]; then - gfs_ver=v12 - RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh - elif [ $yy$mm$dd$hh -lt 2017072000 ]; then - gfs_ver=v13 - RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh - elif [ $yy$mm$dd$hh -lt 2019061200 ]; then - gfs_ver=v14 - RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh - elif [ $yy$mm$dd$hh -lt 2021031700 ]; then - gfs_ver=v15 - RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh - fi -fi - -export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF -export LEVS gfs_ver +export COMPONENT="atmos" +export gfs_ver=${gfs_ver:-v16} +export RUNICSH=${RUNICSH:-${GDASINIT_DIR}/run_v16.chgres.sh} # Check if init is needed and run if so if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then diff --git a/parm/config/config.init b/parm/config/config.init index 212fa422df3..a9932028a56 100755 --- a/parm/config/config.init +++ b/parm/config/config.init @@ -18,7 +18,7 @@ export CRES_HIRES=$CASE export CRES_ENKF=$CASE_ENKF if [ ${RETRO:-"NO"} = "YES" ]; then - export GETICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh + export RUNICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh fi if [ ${RETRO:-"NO"} = "NO" ]; then # Operational input From efdf917a843cc301a2fe7d5386aea1e29dd02c46 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 5 Mar 2021 21:29:22 +0000 Subject: [PATCH 12/40] Update in wave prep script and add ability to pull warm starts - Update getic job to detect warm start config info and pull warm start RESTART files off HPSS and place in ROTDIR; just for retro ICs, will add ops after implementation - Add check to JGLOBAL_WAVE_PREP for prior cycle rtofs ROTDIR symlink; normally cycled mode would have already created this during prior cycle so added check for use in free-forecast mode Refs: #178 --- jobs/JGLOBAL_WAVE_PREP | 4 ++++ jobs/rocoto/getic.sh | 36 +++++++++++++++++++++++++++++++++--- parm/config/config.getic | 17 +++++++++++++---- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/jobs/JGLOBAL_WAVE_PREP b/jobs/JGLOBAL_WAVE_PREP index 4fe921c9aba..138a3bf64d8 100755 --- a/jobs/JGLOBAL_WAVE_PREP +++ b/jobs/JGLOBAL_WAVE_PREP @@ -77,6 +77,10 @@ else if [ ! -L $ROTDIR/${WAVECUR_DID}.${RPDY} ]; then # Check if symlink already exists in ROTDIR $NLN $DMPDIR/${WAVECUR_DID}.${RPDY} $ROTDIR/${WAVECUR_DID}.${RPDY} fi + BRPDY=`$NDATE -24 ${RPDY}00 | cut -c1-8` + if [ ! -L $ROTDIR/${WAVECUR_DID}.${BRPDY} ]; then # Check if symlink already exists in ROTDIR + $NLN $DMPDIR/${WAVECUR_DID}.${BRPDY} $ROTDIR/${WAVECUR_DID}.${BRPDY} + fi if [ ! -L $ROTDIR/${CDUMP}.${PDY}/${cyc}/atmos/${WAVICEFILE} ]; then # Check if symlink already exists in ROTDIR $NLN $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} $ROTDIR/$CDUMP.${PDY}/$cyc/atmos/${WAVICEFILE} fi diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index 6523acce5cf..a890b0de200 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -52,9 +52,39 @@ export GETICSH=${GETICSH:-${GDASINIT_DIR}/get_v16.data.sh} # Run get data script if [ ! -d $EXTRACT_DIR ]; then mkdir -p $EXTRACT_DIR ; fi -sh ${GETICSH} ${CDUMP} -status=$? -[[ $status -ne 0 ]] && exit $status + +# Check if init is needed and run if so +if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then + # Pull RESTART files off HPSS + + if [ ! -d $ROTDIR ]; then mkdir $ROTDIR ; fi + cd $ROTDIR + + if [ ${RETRO:-"NO"} = "YES" ]; then # Retrospective parallel input + + # Pull prior cycle restart files + BDATE=`$NDATE -06 ${yy}${mm}${dd}${hh}` + htar -xvf ${HPSSDIR}/${BDATE}/gdas_restartb.tar + status=$? + [[ $status -ne 0 ]] && exit $status + + # Pull current cycle restart files + htar -xvf ${HPSSDIR}/${CDATE}/gfs_restarta.tar + status=$? + [[ $status -ne 0 ]] && exit $status + +# else # Opertional input +# # ADD AFTER IMPLEMENTATION + fi + +else + + # Run UFS_UTILS GETICSH + sh ${GETICSH} ${CDUMP} + status=$? + [[ $status -ne 0 ]] && exit $status + +fi # Pull pgbanl file for verification/archival - v14+ if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then diff --git a/parm/config/config.getic b/parm/config/config.getic index ec396849ae7..b86f9c4aebb 100755 --- a/parm/config/config.getic +++ b/parm/config/config.getic @@ -17,11 +17,20 @@ export GDASINIT_DIR=${UFS_DIR}/util/gdas_init export PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory export GETICSH=${GDASINIT_DIR}/get_v16.data.sh -if [ ${RETRO:-"NO"} = "YES" ]; then +if [ ${RETRO:-"NO"} = "YES" ]; then # Retrospective parallel input export GETICSH=${GDASINIT_DIR}/get_v16retro.data.sh -fi - -if [ ${RETRO:-"NO"} = "NO" ]; then # Operational input + if [[ "$CDATE" -lt "2019060106" ]]; then + HPSSDIR=/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_D/gfsv16/v16retro0e + elif [[ "$CDATE" -lt "2019090100" ]]; then + HPSSDIR=/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_D/gfsv16/v16retro1e + elif [[ "$CDATE" -lt "2019101706" ]]; then + HPSSDIR=/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_D/gfsv16/v16retro2e + elif [[ "$CDATE" -lt "2020122206" ]]; then + HPSSDIR=/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_D/gfsv16/v16rt2 + else + HPSSDIR=/NCEPDEV/emc-global/5year/emc.gloparadev/WCOSS_D/gfsv16/v16rt2n + fi +elif [ ${RETRO:-"NO"} = "NO" ]; then # Operational input # No ENKF data prior to 2012/05/21/00z if [[ "$CDATE" -lt "2012052100" ]]; then set +x From 86b0345a550a7128fd6420befcb793a94411e4db Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 8 Mar 2021 14:09:40 +0000 Subject: [PATCH 13/40] Add pull of IAU increment files off HPSS for free-forecast Refs: #178 --- jobs/rocoto/getic.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index a890b0de200..f8f06cda999 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -73,6 +73,11 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then status=$? [[ $status -ne 0 ]] && exit $status + # Pull IAU increment files + htar -xvf ${HPSSDIR}/${CDATE}/gfs_netcdfa.tar + status=$? + [[ $status -ne 0 ]] && exit $status + # else # Opertional input # # ADD AFTER IMPLEMENTATION fi From 364e46ff2acabed7bd9f6cb07ca10aa19033a980 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 9 Mar 2021 16:08:57 +0000 Subject: [PATCH 14/40] Add pull of gdas wave restart files in getic.sh if DO_WAVE=YES Refs: #178 --- jobs/rocoto/getic.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index f8f06cda999..33ca3789d99 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -68,6 +68,13 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then status=$? [[ $status -ne 0 ]] && exit $status + if [ $DO_WAVE = "YES" ]; then + # Pull prior cycle wave restart files + htar -xvf ${HPSSDIR}/${BDATE}/gdaswave_restart.tar + status=$? + [[ $status -ne 0 ]] && exit $status + fi + # Pull current cycle restart files htar -xvf ${HPSSDIR}/${CDATE}/gfs_restarta.tar status=$? From fcd720edef1fe242c27d543f445bb3b4ee4fe9ec Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 9 Mar 2021 17:54:51 +0000 Subject: [PATCH 15/40] Remove unneeded UFS_UTILS symlinks in link_fv3gfs.sh Refs: #178 --- sorc/link_fv3gfs.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh index 876bf9456a4..451b174692e 100755 --- a/sorc/link_fv3gfs.sh +++ b/sorc/link_fv3gfs.sh @@ -77,8 +77,7 @@ cd ${pwd}/../ush ||exit 8 $LINK ../sorc/gfs_post.fd/ush/$file . done for file in emcsfc_ice_blend.sh fv3gfs_driver_grid.sh fv3gfs_make_orog.sh global_cycle_driver.sh \ - emcsfc_snow.sh fv3gfs_filter_topo.sh global_chgres_driver.sh global_cycle.sh \ - fv3gfs_chgres.sh fv3gfs_make_grid.sh global_chgres.sh ; do + emcsfc_snow.sh fv3gfs_filter_topo.sh global_cycle.sh fv3gfs_make_grid.sh ; do $LINK ../sorc/ufs_utils.fd/ush/$file . done for file in gldas_archive.sh gldas_forcing.sh gldas_get_data.sh gldas_process_data.sh gldas_liscrd.sh gldas_post.sh ; do @@ -310,13 +309,10 @@ cd ${pwd}/../sorc || exit 8 $SLINK gfs_post.fd/sorc/ncep_post.fd gfs_ncep_post.fd - $SLINK ufs_utils.fd/sorc/fre-nctools.fd/tools/shave.fd shave.fd - for prog in filter_topo fregrid make_hgrid make_solo_mosaic ; do + for prog in fregrid make_hgrid make_solo_mosaic ; do $SLINK ufs_utils.fd/sorc/fre-nctools.fd/tools/$prog ${prog}.fd done - for prog in global_cycle.fd nemsio_read.fd nemsio_chgdate.fd \ - emcsfc_ice_blend.fd nst_tf_chg.fd \ - emcsfc_snow2mdl.fd global_chgres.fd nemsio_get.fd orog.fd ;do + for prog in global_cycle.fd emcsfc_ice_blend.fd emcsfc_snow2mdl.fd ; do $SLINK ufs_utils.fd/sorc/$prog $prog done From 4f7243bc68164faf0008893467c5a2ea53d216ae Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 9 Mar 2021 18:07:23 +0000 Subject: [PATCH 16/40] Remove global_chgres exec from link_fv3gfs.sh; no longer exists from UFS_UTILS Refs: #178 --- sorc/link_fv3gfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh index 451b174692e..f79ccc01b38 100755 --- a/sorc/link_fv3gfs.sh +++ b/sorc/link_fv3gfs.sh @@ -233,7 +233,7 @@ if [ -d ${pwd}/gfs_wafs.fd ]; then fi for ufs_utilsexe in \ - emcsfc_ice_blend emcsfc_snow2mdl global_chgres global_cycle ; do + emcsfc_ice_blend emcsfc_snow2mdl global_cycle ; do [[ -s $ufs_utilsexe ]] && rm -f $ufs_utilsexe $LINK ../sorc/ufs_utils.fd/exec/$ufs_utilsexe . done From 5c77500bc4742a840a26f05aedffe2aca3e15ec7 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 22 Mar 2021 15:44:40 +0000 Subject: [PATCH 17/40] Add exit in init.sh when C768 v16 warm starts detected Exit init.sh and init job when high res C768 v16 warm starts are detected and running chgres_cube is not needed. Refs: #178 --- jobs/rocoto/init.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh index 2fb81b268b6..f7bc323aaf9 100755 --- a/jobs/rocoto/init.sh +++ b/jobs/rocoto/init.sh @@ -52,6 +52,7 @@ export RUNICSH=${RUNICSH:-${GDASINIT_DIR}/run_v16.chgres.sh} # Check if init is needed and run if so if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then echo "Detected v16 C768 warm starts, will not run init. Exiting..." + exit 0 else # Run chgres_cube if [ ! -d $OUTDIR ]; then mkdir -p $OUTDIR ; fi From 48ea7abd9fe4cfd0e774c92383948c9bb477a905 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 12 Apr 2021 17:10:59 +0000 Subject: [PATCH 18/40] Low resolution resource adjustments - C96, C192, and C384 config.fv3 resource updates for successful jobs - provided by Jun Wang and Fanglin Yang - tested on Hera Refs: #178 --- parm/config/config.fv3 | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 index eca11a0f815..1c4025f84c6 100755 --- a/parm/config/config.fv3 +++ b/parm/config/config.fv3 @@ -53,7 +53,7 @@ case $case_in in export WRTIOBUF="1M" ;; "C96") - export DELTIM=900 + export DELTIM=720 export layout_x=6 export layout_y=4 export layout_x_gfs=6 @@ -64,10 +64,11 @@ case $case_in in export nth_fv3_gfs=1 export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 - export WRTTASK_PER_GROUP=6 + export WRTTASK_PER_GROUP=$npe_node_max export WRITE_GROUP_GFS=1 export WRTTASK_PER_GROUP_GFS=$npe_node_max export WRTIOBUF="4M" + export n_split=6 ;; "C192") export DELTIM=450 @@ -88,14 +89,14 @@ case $case_in in ;; "C384") export DELTIM=240 - export layout_x=8 + export layout_x=6 export layout_y=8 export layout_x_gfs=6 - export layout_y_gfs=6 + export layout_y_gfs=8 export npe_wav=140 export npe_wav_gfs=140 - export nth_fv3=1 - export nth_fv3_gfs=1 + export nth_fv3=2 + export nth_fv3_gfs=2 export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 export WRTTASK_PER_GROUP=$npe_node_max From 6b6b9ed3949c2e3ca3d697ffd3bb3a65ec728744 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 12 Apr 2021 17:13:06 +0000 Subject: [PATCH 19/40] getic and init job updates for v16 in ops - update GFSv16 implementation cycle in config if-blocks to be 2021032100 - add note about missing ops data on HPSS for 2021032106 cycle - add run_v16.chgres.sh as default RUNICSH script Refs: #178 --- parm/config/config.getic | 10 +++++++++- parm/config/config.init | 13 +++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/parm/config/config.getic b/parm/config/config.getic index b86f9c4aebb..90b9660716b 100755 --- a/parm/config/config.getic +++ b/parm/config/config.getic @@ -45,10 +45,18 @@ elif [ ${RETRO:-"NO"} = "NO" ]; then # Operational input export gfs_ver=v14 export GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh export tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${PDY}_${cyc}.pgrb2_${grid}.tar - elif [[ "$CDATE" -lt "2021031700" ]]; then + elif [[ "$CDATE" -lt "2021032100" ]]; then export gfs_ver=v15 export GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh export tarball=com_gfs_prod_gfs.${PDY}_${cyc}.gfs_pgrb2.tar + elif [[ "$CDATE" -lt "2021032106" ]]; then + # The way the v16 switch over was done, there is no complete + # set of v16 or v15 data for 2021032100. And although + # v16 was officially implemented 2021032212, the v16 prod + # tarballs were archived starting 2021032106. + set +x + echo FATAL ERROR: NO V15 OR V16 DATA FOR 2021032100 + exit 1 fi fi diff --git a/parm/config/config.init b/parm/config/config.init index a9932028a56..d1d504c7c06 100755 --- a/parm/config/config.init +++ b/parm/config/config.init @@ -17,7 +17,8 @@ export GDASINIT_DIR=${UFS_DIR}/util/gdas_init export CRES_HIRES=$CASE export CRES_ENKF=$CASE_ENKF -if [ ${RETRO:-"NO"} = "YES" ]; then +export RUNICSH=${GDASINIT_DIR}/run_v16.chgres.sh +if [ ${RETRO:-"NO"} = "YES" || $CDUMP = "gdas" ]; then export RUNICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh fi @@ -35,9 +36,17 @@ if [ ${RETRO:-"NO"} = "NO" ]; then # Operational input elif [[ "$CDATE" -lt "2019061200" ]]; then export gfs_ver=v14 export RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh - elif [[ "$CDATE" -lt "2021031700" ]]; then + elif [[ "$CDATE" -lt "2021032100" ]]; then export gfs_ver=v15 export RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh + elif [[ "$CDATE" -lt "2021032106" ]]; then + # The way the v16 switch over was done, there is no complete + # set of v16 or v15 data for 2021032100. And although + # v16 was officially implemented 2021032212, the v16 prod + # tarballs were archived starting 2021032106. + set +x + echo FATAL ERROR: NO V15 OR V16 DATA FOR 2021032100 + exit 1 fi fi From 151e0b56b6e38f40449c9eb0807ffa92b6aa46f9 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 25 May 2021 20:41:32 +0000 Subject: [PATCH 20/40] Correct if-statement syntax in config.init Refs: #178 --- parm/config/config.init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.init b/parm/config/config.init index d1d504c7c06..eeb04a7d409 100755 --- a/parm/config/config.init +++ b/parm/config/config.init @@ -18,7 +18,7 @@ export CRES_HIRES=$CASE export CRES_ENKF=$CASE_ENKF export RUNICSH=${GDASINIT_DIR}/run_v16.chgres.sh -if [ ${RETRO:-"NO"} = "YES" || $CDUMP = "gdas" ]; then +if [ "${RETRO:-"NO"}" = "YES" ] || [ "$CDUMP" = "gdas" ]; then export RUNICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh fi From 824ff7254fba04a0b3726316d300f9fa6282c937 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 25 May 2021 20:42:25 +0000 Subject: [PATCH 21/40] Change UFS_UTILS version to feature branch for testing Refs: #178 --- sorc/checkout.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 62d6fcd6d04..34c80c36dcf 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -68,9 +68,11 @@ fi echo ufs_utils checkout ... if [[ ! -d ufs_utils.fd ]] ; then rm -f ${topdir}/checkout-ufs_utils.log - git clone https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1 + #git clone https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1 + git clone --recursive https://github.com/GeorgeGayno-NOAA/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1 cd ufs_utils.fd - git checkout ops-gfsv16.0.0 + #git checkout ops-gfsv16.0.0 + git checkout feature/gdas_init.v16 cd ${topdir} else echo 'Skip. Directory ufs_utils.fd already exists.' From 499f217f92b5bb0fad9e71de221937d9973046ce Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 1 Jun 2021 15:49:41 +0000 Subject: [PATCH 22/40] Resolve bug in wavepostpnt jobs when DOBLL_WAV=NO - change DOBNDPNT_WAV=YES and CFP_MP=YES if-block to only prepare cmdtarfile for CFP when DOBLL_WAV=YES to avoid scenario where only a single command is added to cmdtarfile Refs: #178 --- scripts/exgfs_wave_post_pnt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 45f74796dd1..4ee72c3c2d7 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -624,7 +624,7 @@ if [ ${CFP_MP:-"NO"} = "YES" ]; then nm=0; fi - if [ ${CFP_MP:-"NO"} = "YES" ]; then + if [ ${CFP_MP:-"NO"} = "YES" ] && [ "$DOBLL_WAV" = "YES" ]; then if [ "$DOBNDPNT_WAV" = YES ]; then if [ "$DOSPC_WAV" = YES ]; then echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile From 127723adfde2b7b7c79bd4d346cafcb1f265dc1c Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 1 Jun 2021 16:44:37 +0000 Subject: [PATCH 23/40] getic job updates to add OPS_RES variable and pull from operational tarballs for warm starting - add OPS_RES variable to define current operational resolution; use variable where needed now - updated getic job script to use OPS_RES variable in if-block check - updated getic job script with new operational tarballs to pull for warm starting high res runs Refs: #178 --- jobs/rocoto/getic.sh | 21 ++++++++++++++++----- parm/config/config.base.emc.dyn | 1 + 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index 33ca3789d99..89c207a281e 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -40,6 +40,11 @@ export yy=$(echo $CDATE | cut -c1-4) export mm=$(echo $CDATE | cut -c5-6) export dd=$(echo $CDATE | cut -c7-8) export hh=${cyc:-$(echo $CDATE | cut -c9-10)} +export GDATE=$($NDATE -${assim_freq:-"06"} $CDATE) +export gyy=$(echo $GDATE | cut -c1-4) +export gmm=$(echo $GDATE | cut -c5-6) +export gdd=$(echo $GDATE | cut -c7-8) +export ghh=$(echo $GDATE | cut -c9-10) export DATA=${DATA:-${DATAROOT}/init} export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR} @@ -47,16 +52,17 @@ export WORKDIR=${WORKDIR:-$DATA} export OUTDIR=${OUTDIR:-$ROTDIR} export PRODHPSSDIR=${PRODHPSSDIR:-/NCEPPROD/hpssprod/runhistory} export COMPONENT="atmos" -export gfs_ver=${gfs_ver:-v16} +export gfs_ver=${gfs_ver:-"v16"} +export OPS_RES=${OPS_RES:-"C768"} export GETICSH=${GETICSH:-${GDASINIT_DIR}/get_v16.data.sh} # Run get data script if [ ! -d $EXTRACT_DIR ]; then mkdir -p $EXTRACT_DIR ; fi # Check if init is needed and run if so -if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then - # Pull RESTART files off HPSS +if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; then + # Pull RESTART files off HPSS if [ ! -d $ROTDIR ]; then mkdir $ROTDIR ; fi cd $ROTDIR @@ -85,8 +91,13 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then status=$? [[ $status -ne 0 ]] && exit $status -# else # Opertional input -# # ADD AFTER IMPLEMENTATION + else # Opertional input - warm starts + + # Pull CDATE gfs restart tarball + htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_restart.tar + # Pull GDATE gdas restart tarball + htar -xvf ${PRODHPSSDIR}/rh${gyy}/${gyy}${gmm}/${gyy}${gmm}${gdd}/com_gfs_prod_gfs.${gyy}${gmm}${gdd}_${ghh}.gfs_restart.tar + fi else diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index 26be95adc60..3016b983e42 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -140,6 +140,7 @@ export DBNROOT=${DBNROOT:-${UTILROOT}/fakedbn} export LEVS=128 export CASE="@CASECTL@" export CASE_ENKF="@CASEENS@" +export OPS_RES="C768" # Run with CCPP physics export RUN_CCPP="YES" From e62cefb3943a87a90fd40234f08845534df818a1 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 1 Jun 2021 16:50:28 +0000 Subject: [PATCH 24/40] Correct gdas operational tarball name in getic job script Refs: #178 --- jobs/rocoto/getic.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index 89c207a281e..f62b4494a5f 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -96,8 +96,7 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; the # Pull CDATE gfs restart tarball htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_restart.tar # Pull GDATE gdas restart tarball - htar -xvf ${PRODHPSSDIR}/rh${gyy}/${gyy}${gmm}/${gyy}${gmm}${gdd}/com_gfs_prod_gfs.${gyy}${gmm}${gdd}_${ghh}.gfs_restart.tar - + htar -xvf ${PRODHPSSDIR}/rh${gyy}/${gyy}${gmm}/${gyy}${gmm}${gdd}/com_gfs_prod_gdas.${gyy}${gmm}${gdd}_${ghh}.gdas_restart.tar fi else From f0afb91bae738413539ee2f1fdc63b79e93b3e52 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Wed, 2 Jun 2021 14:22:43 +0000 Subject: [PATCH 25/40] Change BDATE to GDATE in getic.sh Refs: #178 --- jobs/rocoto/getic.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index f62b4494a5f..d17089e095d 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -69,14 +69,13 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; the if [ ${RETRO:-"NO"} = "YES" ]; then # Retrospective parallel input # Pull prior cycle restart files - BDATE=`$NDATE -06 ${yy}${mm}${dd}${hh}` - htar -xvf ${HPSSDIR}/${BDATE}/gdas_restartb.tar + htar -xvf ${HPSSDIR}/${GDATE}/gdas_restartb.tar status=$? [[ $status -ne 0 ]] && exit $status if [ $DO_WAVE = "YES" ]; then # Pull prior cycle wave restart files - htar -xvf ${HPSSDIR}/${BDATE}/gdaswave_restart.tar + htar -xvf ${HPSSDIR}/${GDATE}/gdaswave_restart.tar status=$? [[ $status -ne 0 ]] && exit $status fi From e2ac5872ca482eb335bc6294e00594dc3a806718 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Wed, 2 Jun 2021 14:23:35 +0000 Subject: [PATCH 26/40] Update WCOSS-Dell section of machine-setup.sh - needed cmake module loaded for building - added cmake module load and prereq stack loads Refs: #178 --- sorc/machine-setup.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index c39cbfb2722..f0f5586c235 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -95,6 +95,9 @@ elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then fi target=wcoss_dell_p3 module purge + module use /gpfs/dell2/usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles + module load hpc/1.1.0 + module load cmake/3.20.0 ##--------------------------------------------------------------------------- From 771782f109de15f9f0214bcceb852b51bb4b0e02 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 3 Jun 2021 13:09:42 -0500 Subject: [PATCH 27/40] Replace Orion checks with hpssarch checks for getic job - add hpssarch variable to setup_workflow_fcstonly.py - replace Orion checks with hpssarch=YES checks Refs: #178 --- ush/rocoto/setup_workflow_fcstonly.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index 079a4ad3eca..66f553b8be7 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -91,6 +91,7 @@ def get_definitions(base): machine = base.get('machine', wfu.detectMachine()) scheduler = wfu.get_scheduler(machine) + hpssarch = base.get('HPSSARCH', 'NO').upper() strings = [] @@ -131,7 +132,7 @@ def get_definitions(base): strings.append('\t\n' % scheduler) strings.append('\n') strings.append('\t\n') - strings.append('\t\n') + strings.append('\t\n' % base['HPSSARCH']) strings.append('\n') strings.append('\t\n') strings.append('\t\n') @@ -236,6 +237,7 @@ def get_workflow(dict_configs, cdump='gdas'): base = dict_configs['base'] machine = base.get('machine', wfu.detectMachine()) + hpssarch = base.get('HPSSARCH', 'NO').upper() do_wave = base.get('DO_WAVE', 'NO').upper() do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() @@ -247,7 +249,7 @@ def get_workflow(dict_configs, cdump='gdas'): tasks = [] # getic - if machine not in ['ORION']: + if hpssarch in ['YES']: deps = [] data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' dep_dict = {'type':'data', 'data':data} @@ -280,7 +282,7 @@ def get_workflow(dict_configs, cdump='gdas'): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - if machine not in ['ORION']: + if hpssarch in ['YES']: deps = [] dep_dict = {'type': 'task', 'name': '%sgetic' % cdump} deps.append(rocoto.add_dependency(dep_dict)) @@ -288,7 +290,7 @@ def get_workflow(dict_configs, cdump='gdas'): deps = [] deps.append(dependencies) - if machine not in ['ORION']: + if hpssarch in ['YES']: deps.append(dependencies2) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) From bfb0a33305a46d87f62c4a5c8bd7799a60362678 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 3 Jun 2021 13:13:08 -0500 Subject: [PATCH 28/40] Remove RUN_CCPP option, force CCPP now - remove RUN_CCPP case option from build_all.sh - remove RUN_CCPP option in build_fv3.sh, force CCPP build now - remove pre-CCPP GFS ops tag checkout option in checkout.sh, forcing CCPP hash checkout now Refs: #178 --- sorc/build_all.sh | 11 ----------- sorc/build_fv3.sh | 13 ++++++------- sorc/checkout.sh | 10 +--------- 3 files changed, 7 insertions(+), 27 deletions(-) diff --git a/sorc/build_all.sh b/sorc/build_all.sh index acba8d6a2cc..6c027efc792 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -9,16 +9,6 @@ set +x # Anything other than "true" will use libraries locally. #------------------------------------ -while getopts "c" option; -do - case $option in - c) - echo "Received -c flag, check out ufs-weather-model develop branch with CCPP physics" - RUN_CCPP="YES" - ;; - esac -done - export USE_PREINST_LIBS="true" #------------------------------------ @@ -61,7 +51,6 @@ err=0 #------------------------------------ $Build_fv3gfs && { echo " .... Building fv3 .... " -export RUN_CCPP=${RUN_CCPP:-"NO"} ./build_fv3.sh > $logs_dir/build_fv3.log 2>&1 rc=$? if [[ $rc -ne 0 ]] ; then diff --git a/sorc/build_fv3.sh b/sorc/build_fv3.sh index 2c3efdadec3..376967f3740 100755 --- a/sorc/build_fv3.sh +++ b/sorc/build_fv3.sh @@ -23,10 +23,9 @@ cd fv3gfs.fd/ FV3=$( pwd -P )/FV3 cd tests/ if [ ! -d ../NEMS/exe ]; then mkdir ../NEMS/exe ; fi -if [ ${RUN_CCPP:-${1:-"YES"}} = "NO" ]; then - ./compile.sh "$FV3" "$target" "WW3=Y 32BIT=Y" 1 - mv -f fv3_1.exe ../NEMS/exe/global_fv3gfs.x -else - ./compile.sh "$target" "APP=ATM CCPP=Y 32BIT=Y SUITES=FV3_GFS_v16,FV3_GFS_v16_RRTMGP" 2 YES NO - mv -f fv3_2.exe ../NEMS/exe/global_fv3gfs.x -fi + +#./compile.sh "$FV3" "$target" "WW3=Y 32BIT=Y" 1 +#mv -f fv3_1.exe ../NEMS/exe/global_fv3gfs.x + +./compile.sh "$target" "APP=ATM CCPP=Y 32BIT=Y SUITES=FV3_GFS_v16,FV3_GFS_v16_RRTMGP" 2 YES NO +mv -f fv3_2.exe ../NEMS/exe/global_fv3gfs.x diff --git a/sorc/checkout.sh b/sorc/checkout.sh index ed3b6b1c563..eda2d007355 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -9,10 +9,6 @@ do echo "Received -o flag for optional checkout of GTG, will check out GTG with EMC_post" checkout_gtg="YES" ;; - c) - echo "Received -c flag, check out ufs-weather-model develop branch with CCPP physics" - run_ccpp="YES" - ;; :) echo "option -$OPTARG needs an argument" ;; @@ -31,11 +27,7 @@ if [[ ! -d fv3gfs.fd ]] ; then rm -f ${topdir}/checkout-fv3gfs.log git clone https://github.com/ufs-community/ufs-weather-model fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1 cd fv3gfs.fd - if [ ${run_ccpp:-"NO"} = "NO" ]; then - git checkout GFS.v16.0.16 - else - git checkout 9350745855aebe0790813e0ed2ba5ad680e3f75c - fi + git checkout 9350745855aebe0790813e0ed2ba5ad680e3f75c git submodule update --init --recursive cd ${topdir} else From fe448ff29aedca6a40a748ee47ae001b1639188c Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 3 Jun 2021 19:32:15 +0000 Subject: [PATCH 29/40] Set DO_WAVE to NO - turning off waves by default in config.base Refs: #178 --- parm/config/config.base.emc.dyn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index 3016b983e42..e68d0c0b0ab 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -226,7 +226,7 @@ export DO_GLDAS="YES" export gldas_cyc=00 # run wave component -export DO_WAVE="YES" +export DO_WAVE="NO" if [[ "$SDATE" -lt "2019020100" ]]; then DO_WAVE="NO" ; fi # no rtofs in GDA export WAVE_CDUMP="both" From 1cdc2a44c6d29cfb8f06a50238ea29d28642c1f1 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 8 Jun 2021 15:56:46 +0000 Subject: [PATCH 30/40] Add cmake module load to Hera block in machine-setup.sh Refs: #178 --- sorc/machine-setup.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index f0f5586c235..5f28d6abaa9 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -45,11 +45,10 @@ elif [[ -d /scratch1 ]] ; then fi target=hera module purge - module load intel - module load impi - export NCEPLIBS=/scratch2/NCEPDEV/nwprod/NCEPLIBS - module use $NCEPLIBS/modulefiles - #export WRFPATH=$NCEPLIBS/wrf.shared.new/v1.1.1/src + module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack + module load hpc/1.1.0 + module load cmake/3.20.0 + export myFC=mpiifort export FCOMP=mpiifort From 898a43fb74005e969ecb5baf28ecf2d51f7c5dda Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 8 Jun 2021 15:57:32 +0000 Subject: [PATCH 31/40] Adjust getic/init job scripts - adjust EXTRACT_DIR, DATA, and ROTDIR usage in getic and init jobs - remove duplicate pgb file pull from init job Refs: #178 --- jobs/rocoto/getic.sh | 25 ++++++++++++++++--------- jobs/rocoto/init.sh | 12 ------------ 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index d17089e095d..9f0f9277348 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -46,26 +46,23 @@ export gmm=$(echo $GDATE | cut -c5-6) export gdd=$(echo $GDATE | cut -c7-8) export ghh=$(echo $GDATE | cut -c9-10) -export DATA=${DATA:-${DATAROOT}/init} -export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR} -export WORKDIR=${WORKDIR:-$DATA} -export OUTDIR=${OUTDIR:-$ROTDIR} +export DATA=${DATA:-${DATAROOT}/getic} +export EXTRACT_DIR=${DATA:-$EXTRACT_DIR} export PRODHPSSDIR=${PRODHPSSDIR:-/NCEPPROD/hpssprod/runhistory} export COMPONENT="atmos" export gfs_ver=${gfs_ver:-"v16"} export OPS_RES=${OPS_RES:-"C768"} export GETICSH=${GETICSH:-${GDASINIT_DIR}/get_v16.data.sh} -# Run get data script +# Create ROTDIR/EXTRACT_DIR +if [ ! -d $ROTDIR ]; then mkdir -p $ROTDIR ; fi if [ ! -d $EXTRACT_DIR ]; then mkdir -p $EXTRACT_DIR ; fi +cd $EXTRACT_DIR # Check if init is needed and run if so if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; then # Pull RESTART files off HPSS - if [ ! -d $ROTDIR ]; then mkdir $ROTDIR ; fi - cd $ROTDIR - if [ ${RETRO:-"NO"} = "YES" ]; then # Retrospective parallel input # Pull prior cycle restart files @@ -107,6 +104,10 @@ else fi +# Move extracted data to ROTDIR +if [ ! -d ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh} ]; then mkdir -p ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh} ; fi +mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/* ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh} + # Pull pgbanl file for verification/archival - v14+ if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then cd $EXTRACT_DIR @@ -114,10 +115,16 @@ if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then do file=gfs.t${hh}z.pgrb2.${grid}.anl htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file} - mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file} + mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file} done fi +########################################## +# Remove the Temporary working directory +########################################## +cd $DATAROOT +[[ $KEEPDATA = "NO" ]] && rm -rf $DATA + ############################################################### # Exit out cleanly exit 0 diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh index f7bc323aaf9..6b70842821f 100755 --- a/jobs/rocoto/init.sh +++ b/jobs/rocoto/init.sh @@ -61,18 +61,6 @@ else [[ $status -ne 0 ]] && exit $status fi -# Check for pgbanl files and move -if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then - cd $EXTRACT_DIR - for grid in 0p25 0p50 1p00 - do - file=gfs.t${hh}z.pgrb2.${grid}.anl - if [ -f ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ]; then - mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file} - fi - done -fi - ########################################## # Remove the Temporary working directory ########################################## From 4c858c0f059abd0ff4c937c7bab692d7b13a8163 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Tue, 8 Jun 2021 13:05:08 -0500 Subject: [PATCH 32/40] Adjust comments in getic job script Refs: #178 --- jobs/rocoto/getic.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index 9f0f9277348..f6692591794 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -59,8 +59,8 @@ if [ ! -d $ROTDIR ]; then mkdir -p $ROTDIR ; fi if [ ! -d $EXTRACT_DIR ]; then mkdir -p $EXTRACT_DIR ; fi cd $EXTRACT_DIR -# Check if init is needed and run if so -if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; then +# Check version, cold/warm start, and resolution +if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; then # Pull warm start ICs - no chgres # Pull RESTART files off HPSS if [ ${RETRO:-"NO"} = "YES" ]; then # Retrospective parallel input @@ -95,7 +95,7 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; the htar -xvf ${PRODHPSSDIR}/rh${gyy}/${gyy}${gmm}/${gyy}${gmm}${gdd}/com_gfs_prod_gdas.${gyy}${gmm}${gdd}_${ghh}.gdas_restart.tar fi -else +else # Pull chgres cube inputs for cold start IC generation # Run UFS_UTILS GETICSH sh ${GETICSH} ${CDUMP} From b52d2f0b82e0e376905c64daa593362396f8da1b Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Tue, 8 Jun 2021 13:06:30 -0500 Subject: [PATCH 33/40] Update UFS_UTILS checkout to ufs_utils_1_4_0 tag - new ufs_utils_1_4_0 tag includes updates to support this branch - From tag release notes: - Update the GDAS initialization utility to ingest GFS v16 data. - Update to process either the GFS or GDAS CDUMPs. Refs: #178 --- sorc/checkout.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sorc/checkout.sh b/sorc/checkout.sh index eda2d007355..9ac4b97ea83 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -60,11 +60,9 @@ fi echo ufs_utils checkout ... if [[ ! -d ufs_utils.fd ]] ; then rm -f ${topdir}/checkout-ufs_utils.log - #git clone https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1 - git clone --recursive https://github.com/GeorgeGayno-NOAA/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1 + git clone https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1 cd ufs_utils.fd - #git checkout ops-gfsv16.0.0 - git checkout feature/gdas_init.v16 + git checkout ufs_utils_1_4_0 cd ${topdir} else echo 'Skip. Directory ufs_utils.fd already exists.' From 18970c716d79922fa02b86165996bb47fb42d3fa Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Wed, 9 Jun 2021 12:04:01 -0500 Subject: [PATCH 34/40] Disconnect archive job in workflow from HPSS access check - remove dependency on HPSS access to check for adding gdas[gfs]arch job to workflow - add HPSSARCH variable to ARCHIVE_TO_HPSS definition in cycled workflow, similar to prior addition to free-forecast script Refs: #178 --- ush/rocoto/setup_workflow.py | 5 ++--- ush/rocoto/setup_workflow_fcstonly.py | 2 -- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index 4a4bfa6bb82..f20a09a2668 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -161,6 +161,7 @@ def get_definitions(base): machine = base.get('machine', wfu.detectMachine()) scheduler = wfu.get_scheduler(machine) + hpssarch = base.get('HPSSARCH', 'NO').upper() strings = [] @@ -198,7 +199,7 @@ def get_definitions(base): strings.append('\t\n' % scheduler) strings.append('\n') strings.append('\t\n') - strings.append('\t\n') + strings.append('\t\n' % base['HPSSARCH']) strings.append('\n') strings.append('\t\n') strings.append('\t\n') @@ -918,8 +919,6 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): deps = [] dep_dict = {'type': 'task', 'name': '%svrfy' % cdump} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'streq', 'left': '&ARCHIVE_TO_HPSS;', 'right': 'YES'} - deps.append(rocoto.add_dependency(dep_dict)) if do_wave in ['Y', 'YES']: dep_dict = {'type': 'task', 'name': '%swavepostsbs' % cdump} deps.append(rocoto.add_dependency(dep_dict)) diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index 66f553b8be7..f25ec53f1c5 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -685,8 +685,6 @@ def get_workflow(dict_configs, cdump='gdas'): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type':'task', 'name':'%svrfy' % cdump} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type':'streq', 'left':'&ARCHIVE_TO_HPSS;', 'right':'YES'} - deps.append(rocoto.add_dependency(dep_dict)) if do_wave in ['Y', 'YES']: dep_dict = {'type': 'task', 'name': '%swavepostsbs' % cdump} deps.append(rocoto.add_dependency(dep_dict)) From 30aefb11dd8130f1d4a1e6eb951e1fa400fdc361 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 10 Jun 2021 15:43:27 +0000 Subject: [PATCH 35/40] Update/fix pull of v14/v15 pgrb2 anl files in getic job Refs: #178 --- jobs/rocoto/getic.sh | 13 ++++++++++--- parm/config/config.getic | 2 -- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index f6692591794..b90412ee3dd 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -110,12 +110,19 @@ mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/* ${ROTDIR}/${CDUMP}.${yy}${mm} # Pull pgbanl file for verification/archival - v14+ if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then - cd $EXTRACT_DIR for grid in 0p25 0p50 1p00 do file=gfs.t${hh}z.pgrb2.${grid}.anl - htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file} - mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file} + if [ $gfs_ver = v14 ]; then + cd $ROTDIR/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT} + export tarball="gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}${hh}.pgrb2_${grid}.tar" + htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${file} + elif [ $gfs_ver = v15 ]; then + cd $EXTRACT_DIR + export tarball="com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar" + htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file} + mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file} + fi done fi diff --git a/parm/config/config.getic b/parm/config/config.getic index 90b9660716b..ced2c051f00 100755 --- a/parm/config/config.getic +++ b/parm/config/config.getic @@ -44,11 +44,9 @@ elif [ ${RETRO:-"NO"} = "NO" ]; then # Operational input elif [[ "$CDATE" -lt "2019061200" ]]; then export gfs_ver=v14 export GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh - export tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${PDY}_${cyc}.pgrb2_${grid}.tar elif [[ "$CDATE" -lt "2021032100" ]]; then export gfs_ver=v15 export GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh - export tarball=com_gfs_prod_gfs.${PDY}_${cyc}.gfs_pgrb2.tar elif [[ "$CDATE" -lt "2021032106" ]]; then # The way the v16 switch over was done, there is no complete # set of v16 or v15 data for 2021032100. And although From db18627960d2def7bbfee30994f4be6586babe1c Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 10 Jun 2021 18:36:10 +0000 Subject: [PATCH 36/40] Increase init job walltime to 30 mins from 15 mins Refs: #178 --- parm/config/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 31cf6f0d659..f404ad8b1b3 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -273,7 +273,7 @@ elif [ $step = "echgres" ]; then elif [ $step = "init" ]; then - export wtime_init="00:15:00" + export wtime_init="00:30:00" export npe_init=24 export nth_init=1 export npe_node_init=6 From 37e7c2e0806678c93a11bae682f48da172940fd6 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 10 Jun 2021 18:36:59 +0000 Subject: [PATCH 37/40] Add OPS_RES variable to init script Refs: #178 --- jobs/rocoto/init.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh index 6b70842821f..d76233f92f5 100755 --- a/jobs/rocoto/init.sh +++ b/jobs/rocoto/init.sh @@ -46,12 +46,13 @@ export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR} export WORKDIR=${WORKDIR:-$DATA} export OUTDIR=${OUTDIR:-$ROTDIR} export COMPONENT="atmos" -export gfs_ver=${gfs_ver:-v16} +export gfs_ver=${gfs_ver:-"v16"} +export OPS_RES=${OPS_RES:-"C768"} export RUNICSH=${RUNICSH:-${GDASINIT_DIR}/run_v16.chgres.sh} # Check if init is needed and run if so -if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then - echo "Detected v16 C768 warm starts, will not run init. Exiting..." +if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; then + echo "Detected v16 $OPS_RES warm starts, will not run init. Exiting..." exit 0 else # Run chgres_cube From 65ff48e049a651e74be8be121163c20957bf1c44 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 10 Jun 2021 18:37:24 +0000 Subject: [PATCH 38/40] Add v16 pgb anl pull to getic script Refs: #178 --- jobs/rocoto/getic.sh | 46 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index b90412ee3dd..5c8e3353b9f 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -105,26 +105,56 @@ else # Pull chgres cube inputs for cold start IC generation fi # Move extracted data to ROTDIR -if [ ! -d ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh} ]; then mkdir -p ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh} ; fi -mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/* ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh} +if [ ! -d ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT} ]; then mkdir -p ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT} ; fi +if [ $gfs_ver = v16 -a $RETRO = "YES" ]; then + mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/* ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT} +else + mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/* ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh} +fi # Pull pgbanl file for verification/archival - v14+ -if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then +if [ $gfs_ver = v14 -o $gfs_ver = v15 -o $gfs_ver = v16 ]; then for grid in 0p25 0p50 1p00 do file=gfs.t${hh}z.pgrb2.${grid}.anl - if [ $gfs_ver = v14 ]; then + + if [ $gfs_ver = v14 ]; then # v14 production source + cd $ROTDIR/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT} export tarball="gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}${hh}.pgrb2_${grid}.tar" htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${file} - elif [ $gfs_ver = v15 ]; then + + elif [ $gfs_ver = v15 ]; then # v15 production source + cd $EXTRACT_DIR export tarball="com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar" htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file} mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file} - fi - done -fi + + elif [ $gfs_ver = v16 ]; then # v16 - determine RETRO or production source next + + if [ $RETRO = "YES" ]; then # Retrospective parallel source + + cd $EXTRACT_DIR + if [ $grid = "0p25" ]; then # anl file spread across multiple tarballs + export tarball="gfsa.tar" + elif [ $grid = "0p50" -o $grid = "1p00" ]; then + export tarball="gfsb.tar" + fi + htar -xvf ${HPSSDIR}/${yy}${mm}${dd}${hh}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file} + mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file} + + else # Production source + + cd $ROTDIR + export tarball="com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar" + htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/atmos/${file} + + fi # RETRO vs production + + fi # Version check + done # grid loop +fi # v14-v16 pgrb anl file pull ########################################## # Remove the Temporary working directory From 921838feb9f6fd3fc3c69579c677ffdd7175a103 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 16 Jun 2021 13:18:35 +0000 Subject: [PATCH 39/40] Remove old compile command from build_fv3.sh Refs: #178 --- sorc/build_fv3.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/sorc/build_fv3.sh b/sorc/build_fv3.sh index 376967f3740..67c28e02ae8 100755 --- a/sorc/build_fv3.sh +++ b/sorc/build_fv3.sh @@ -24,8 +24,5 @@ FV3=$( pwd -P )/FV3 cd tests/ if [ ! -d ../NEMS/exe ]; then mkdir ../NEMS/exe ; fi -#./compile.sh "$FV3" "$target" "WW3=Y 32BIT=Y" 1 -#mv -f fv3_1.exe ../NEMS/exe/global_fv3gfs.x - ./compile.sh "$target" "APP=ATM CCPP=Y 32BIT=Y SUITES=FV3_GFS_v16,FV3_GFS_v16_RRTMGP" 2 YES NO mv -f fv3_2.exe ../NEMS/exe/global_fv3gfs.x From fba8cef5b870d46df1c665d3e9cabb23904ad3cf Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 16 Jun 2021 13:57:14 +0000 Subject: [PATCH 40/40] Remove wave restart pull in getic script Refs: #178 --- jobs/rocoto/getic.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh index 5c8e3353b9f..f08a10a8dd4 100755 --- a/jobs/rocoto/getic.sh +++ b/jobs/rocoto/getic.sh @@ -70,13 +70,6 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; the status=$? [[ $status -ne 0 ]] && exit $status - if [ $DO_WAVE = "YES" ]; then - # Pull prior cycle wave restart files - htar -xvf ${HPSSDIR}/${GDATE}/gdaswave_restart.tar - status=$? - [[ $status -ne 0 ]] && exit $status - fi - # Pull current cycle restart files htar -xvf ${HPSSDIR}/${CDATE}/gfs_restarta.tar status=$?