From 0b28a7f878dac118648692bdd27c48bc1c473de4 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Sat, 4 Mar 2023 16:17:01 +0000 Subject: [PATCH 01/25] refactor UFS-DA atm ens to use python g-w (#1313) --- env/CONTAINER.env | 2 +- env/HERA.env | 12 +- env/JET.env | 12 +- env/ORION.env | 12 +- env/S4.env | 12 +- env/WCOSS2.env | 12 +- jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST | 66 ---- jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP | 66 ---- ...L_RUN => JGLOBAL_ATMENS_ANALYSIS_FINALIZE} | 38 ++- jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE | 55 ++++ jobs/JGLOBAL_ATMENS_ANALYSIS_RUN | 56 ++++ jobs/rocoto/atmensanalpost.sh | 20 -- jobs/rocoto/atmensanalprep.sh | 20 -- jobs/rocoto/atmensanlfinal.sh | 23 ++ jobs/rocoto/atmensanlinit.sh | 24 ++ .../{atmensanalrun.sh => atmensanlrun.sh} | 12 +- parm/config/config.atmensanal | 24 -- parm/config/config.atmensanalpost | 10 - parm/config/config.atmensanalprep | 10 - parm/config/config.atmensanalrun | 14 - parm/config/config.atmensanl | 23 ++ parm/config/config.atmensanlfinal | 10 + parm/config/config.atmensanlinit | 10 + parm/config/config.atmensanlrun | 11 + parm/config/config.resources | 49 +-- scripts/exgdas_global_atmos_ensanal_post.py | 44 --- scripts/exgdas_global_atmos_ensanal_run.sh | 167 ---------- scripts/exglobal_atmens_analysis_finalize.py | 25 ++ .../exglobal_atmens_analysis_initialize.py | 25 ++ scripts/exglobal_atmens_analysis_run.sh | 16 + ush/python/pygfs/task/atmens_analysis.py | 286 ++++++++++++++++++ workflow/applications.py | 10 +- workflow/rocoto/workflow_tasks.py | 47 ++- 33 files changed, 666 insertions(+), 557 deletions(-) delete mode 100755 jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST delete mode 100755 jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP rename jobs/{JGDAS_GLOBAL_ATMOS_ENSANAL_RUN => JGLOBAL_ATMENS_ANALYSIS_FINALIZE} (53%) create mode 100755 jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE create mode 100755 jobs/JGLOBAL_ATMENS_ANALYSIS_RUN delete mode 100755 jobs/rocoto/atmensanalpost.sh delete mode 100755 jobs/rocoto/atmensanalprep.sh create mode 100755 jobs/rocoto/atmensanlfinal.sh create mode 100755 jobs/rocoto/atmensanlinit.sh rename jobs/rocoto/{atmensanalrun.sh => atmensanlrun.sh} (57%) delete mode 100644 parm/config/config.atmensanal delete mode 100644 parm/config/config.atmensanalpost delete mode 100644 parm/config/config.atmensanalprep delete mode 100644 parm/config/config.atmensanalrun create mode 100755 parm/config/config.atmensanl create mode 100755 parm/config/config.atmensanlfinal create mode 100755 parm/config/config.atmensanlinit create mode 100755 parm/config/config.atmensanlrun delete mode 100755 scripts/exgdas_global_atmos_ensanal_post.py delete mode 100755 scripts/exgdas_global_atmos_ensanal_run.sh create mode 100755 scripts/exglobal_atmens_analysis_finalize.py create mode 100755 scripts/exglobal_atmens_analysis_initialize.py create mode 100755 scripts/exglobal_atmens_analysis_run.sh create mode 100644 ush/python/pygfs/task/atmens_analysis.py diff --git a/env/CONTAINER.env b/env/CONTAINER.env index 48014ab313d..bd15d1f5165 100755 --- a/env/CONTAINER.env +++ b/env/CONTAINER.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanalrun atmensanalrun" + echo "atmanalrun atmensanlrun" echo "aeroanlrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" diff --git a/env/HERA.env b/env/HERA.env index f2be166dced..f1d1621b62a 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanalrun atmensanalrun" + echo "atmanalrun atmensanlrun" echo "aeroanlrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" @@ -61,17 +61,17 @@ elif [[ "${step}" = "atmanalrun" ]]; then [[ ${NTHREADS_ATMANAL} -gt ${nth_max} ]] && export NTHREADS_ATMANAL=${nth_max} export APRUN_ATMANAL="${launcher} -n ${npe_atmanalrun}" -elif [[ "${step}" = "atmensanalrun" ]]; then +elif [[ "${step}" = "atmensanlrun" ]]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - nth_max=$((npe_node_max / npe_node_atmensanalrun)) + nth_max=$((npe_node_max / npe_node_atmensanlrun)) - export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-${nth_max}} - [[ ${NTHREADS_ATMENSANAL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANAL=${nth_max} - export APRUN_ATMENSANAL="${launcher} -n ${npe_atmensanalrun}" + export NTHREADS_ATMENSANL=${nth_atmensanlrun:-${nth_max}} + [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} + export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun}" elif [[ "${step}" = "aeroanlrun" ]]; then diff --git a/env/JET.env b/env/JET.env index c04535dad7e..3781f2281ed 100755 --- a/env/JET.env +++ b/env/JET.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanalrun atmensanalrun" + echo "atmanalrun atmensanlrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -51,13 +51,13 @@ elif [[ "${step}" = "atmanalrun" ]]; then [[ ${NTHREADS_ATMANAL} -gt ${nth_max} ]] && export NTHREADS_ATMANAL=${nth_max} export APRUN_ATMANAL="${launcher} ${npe_atmanalrun}" -elif [[ "${step}" = "atmensanalrun" ]]; then +elif [[ "${step}" = "atmensanlrun" ]]; then - nth_max=$((npe_node_max / npe_node_atmensanalrun)) + nth_max=$((npe_node_max / npe_node_atmensanlrun)) - export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-${nth_max}} - [[ ${NTHREADS_ATMENSANAL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANAL=${nth_max} - export APRUN_ATMENSANAL="${launcher} ${npe_atmensanalrun}" + export NTHREADS_ATMENSANL=${nth_atmensanlrun:-${nth_max}} + [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} + export APRUN_ATMENSANL="${launcher} ${npe_atmensanlrun}" elif [[ "${step}" = "anal" ]]; then diff --git a/env/ORION.env b/env/ORION.env index 918748fda37..886378571b3 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanalrun atmensanalrun" + echo "atmanalrun atmensanlrun" echo "aeroanlrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" @@ -61,17 +61,17 @@ elif [[ "${step}" = "atmanalrun" ]]; then [[ ${NTHREADS_ATMANAL} -gt ${nth_max} ]] && export NTHREADS_ATMANAL=${nth_max} export APRUN_ATMANAL="${launcher} -n ${npe_atmanalrun}" -elif [[ "${step}" = "atmensanalrun" ]]; then +elif [[ "${step}" = "atmensanlrun" ]]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - nth_max=$((npe_node_max / npe_node_atmensanalrun)) + nth_max=$((npe_node_max / npe_node_atmensanlrun)) - export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-${nth_max}} - [[ ${NTHREADS_ATMENSANAL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANAL=${nth_max} - export APRUN_ATMENSANAL="${launcher} -n ${npe_atmensanalrun}" + export NTHREADS_ATMENSANL=${nth_atmensanlrun:-${nth_max}} + [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} + export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun}" elif [[ "${step}" = "aeroanlrun" ]]; then diff --git a/env/S4.env b/env/S4.env index 0d11cea02e6..f3bb09a79b8 100755 --- a/env/S4.env +++ b/env/S4.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanalrun atmensanalrun" + echo "atmanalrun atmensanlrun" echo "aeroanlrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" @@ -59,17 +59,17 @@ elif [[ "${step}" = "atmanalrun" ]]; then [[ ${NTHREADS_ATMANAL} -gt ${nth_max} ]] && export NTHREADS_ATMANAL=${nth_max} export APRUN_ATMANAL="${launcher} -n ${npe_atmanalrun}" -elif [[ "${step}" = "atmensanalrun" ]]; then +elif [[ "${step}" = "atmensanlrun" ]]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - nth_max=$((npe_node_max / npe_node_atmensanalrun)) + nth_max=$((npe_node_max / npe_node_atmensanlrun)) - export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-${nth_max}} - [[ ${NTHREADS_ATMENSANAL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANAL=${nth_max} - export APRUN_ATMENSANAL="${launcher} -n ${npe_atmensanalrun}" + export NTHREADS_ATMENSANL=${nth_atmensanlrun:-${nth_max}} + [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} + export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun}" elif [[ "${step}" = "aeroanlrun" ]]; then diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 138f8c38838..ac776051478 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanalrun atmensanalrun" + echo "atmanalrun atmensanlrun" echo "aeroanlrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen esfc efcs epos" @@ -48,17 +48,17 @@ elif [[ "${step}" = "atmanalrun" ]]; then [[ ${NTHREADS_ATMANAL} -gt ${nth_max} ]] && export NTHREADS_ATMANAL=${nth_max} export APRUN_ATMANAL="${launcher} -n ${npe_atmanalrun}" -elif [[ "${step}" = "atmensanalrun" ]]; then +elif [[ "${step}" = "atmensanlrun" ]]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} export APRUNCFP="${launcher} -np \$ncmd ${mpmd_opt}" - nth_max=$((npe_node_max / npe_node_atmensanalrun)) + nth_max=$((npe_node_max / npe_node_atmensanlrun)) - export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-${nth_max}} - [[ ${NTHREADS_ATMENSANAL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANAL=${nth_max} - export APRUN_ATMENSANAL="${launcher} -n ${npe_atmensanalrun}" + export NTHREADS_ATMENSANL=${nth_atmensanlrun:-${nth_max}} + [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} + export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun}" elif [[ "${step}" = "aeroanlrun" ]]; then diff --git a/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST b/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST deleted file mode 100755 index e1d53b552ec..00000000000 --- a/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanalpost" -c "base atmensanal atmensanalpost" - - -############################################## -# Set variables used in the script -############################################## -export CDATE=${CDATE:-${PDY}${cyc}} -export CDUMP=${CDUMP:-${RUN:-"gfs"}} -export COMPONENT="atmos" - -############################################## -# Begin JOB SPECIFIC work -############################################## - -export GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") -gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} -export GDUMP=${GDUMP:-"gdas"} - -export OPREFIX="${CDUMP}.t${cyc}z." -export GPREFIX="${GDUMP}.t${gcyc}z." -export APREFIX="${CDUMP}.t${cyc}z." - -export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}} -export COMOUT_ENS=${COMOUT_ENS:-${ROTDIR}/enkf${CDUMP}.${PDY}/${cyc}/${COMPONENT}} - -mkdir -p ${COMOUT} -mkdir -p ${COMOUT_ENS} - -# COMIN_GES and COMIN_GES_ENS are used in script -export COMIN="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}" -export COMIN_GES="${ROTDIR}/${GDUMP}.${gPDY}/${gcyc}/${COMPONENT}" -export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${gPDY}/${gcyc}/${COMPONENT}" - -# Add UFSDA to PYTHONPATH -export PYTHONPATH=${HOMEgfs}/sorc/gdas.cd/ush/:${PYTHONPATH} - -############################################################### -# Run relevant script - -EXSCRIPT=${GDASPOSTPY:-${HOMEgfs}/scripts/exgdas_global_atmos_ensanal_post.py} -${EXSCRIPT} -status=$? -[[ ${status} -ne 0 ]] && exit ${status} - -############################################## -# End JOB SPECIFIC work -############################################## - -############################################## -# Final processing -############################################## -if [ -e "${pgmout}" ] ; then - cat ${pgmout} -fi - -########################################## -# Remove the Temporary working directory -########################################## -cd ${DATAROOT} -[[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} - -exit 0 diff --git a/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP b/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP deleted file mode 100755 index 7b3ecee7cad..00000000000 --- a/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanalprep" -c "base atmensanal atmensanalprep" - - -############################################## -# Set variables used in the script -############################################## -export CDATE=${CDATE:-${PDY}${cyc}} -export CDUMP=${CDUMP:-${RUN:-"gfs"}} -export COMPONENT="atmos" - -############################################## -# Begin JOB SPECIFIC work -############################################## - -export GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") -gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} -export GDUMP=${GDUMP:-"gdas"} - -export OPREFIX="${CDUMP}.t${cyc}z." -export GPREFIX="${GDUMP}.t${gcyc}z." -export APREFIX="${CDUMP}.t${cyc}z." - -export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}} -export COMOUT_ENS=${COMOUT_ENS:-${ROTDIR}/enkf${CDUMP}.${PDY}/${cyc}/${COMPONENT}} - -mkdir -p ${COMOUT} -mkdir -p ${COMOUT_ENS} - -# COMIN_GES and COMIN_GES_ENS are used in script -export COMIN="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}" -export COMIN_GES="${ROTDIR}/${GDUMP}.${gPDY}/${gcyc}/${COMPONENT}" -export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${gPDY}/${gcyc}/${COMPONENT}" - -# Add UFSDA to PYTHONPATH -export PYTHONPATH=${HOMEgfs}/sorc/gdas.cd/ush/:${PYTHONPATH} - -############################################################### -# Run relevant script - -EXSCRIPT=${GDASPREPPY:-${HOMEgfs}/scripts/exgdas_global_atmos_analysis_prep.py} -${EXSCRIPT} -status=$? -[[ ${status} -ne 0 ]] && exit ${status} - -############################################## -# End JOB SPECIFIC work -############################################## - -############################################## -# Final processing -############################################## -if [ -e "${pgmout}" ] ; then - cat ${pgmout} -fi - -########################################## -# Remove the Temporary working directory -########################################## -cd ${DATAROOT} -[[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} - -exit 0 diff --git a/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN b/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE similarity index 53% rename from jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN rename to jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE index 45368d51ff6..bb7288ca369 100755 --- a/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE @@ -1,21 +1,23 @@ -#!/bin/bash +#! /usr/bin/env bash source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanalrun" -c "base atmensanal atmensanalrun" - +export WIPE_DATA="NO" +export DATA=${DATA:-${DATAROOT}/${RUN}atmensanl_${cyc}} +source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlfinal" -c "base atmensanl atmensanlfinal" ############################################## # Set variables used in the script ############################################## export CDATE=${CDATE:-${PDY}${cyc}} export CDUMP=${CDUMP:-${RUN:-"gfs"}} -export COMPONENT="atmos" + ############################################## # Begin JOB SPECIFIC work ############################################## -export GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") +GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") +export GDATE gPDY=${GDATE:0:8} export gcyc=${GDATE:8:2} export GDUMP=${GDUMP:-"gdas"} @@ -24,27 +26,21 @@ export OPREFIX="${CDUMP}.t${cyc}z." export GPREFIX="${GDUMP}.t${gcyc}z." export APREFIX="${CDUMP}.t${cyc}z." -export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}} -export COMOUT_ENS=${COMOUT_ENS:-${ROTDIR}/enkf${CDUMP}.${PDY}/${cyc}/${COMPONENT}} +export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}} -mkdir -p ${COMOUT} -mkdir -p ${COMOUT_ENS} +mkdir -p "${COMOUT}" # COMIN_GES and COMIN_GES_ENS are used in script -export COMIN="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}" -export COMIN_GES="${ROTDIR}/${GDUMP}.${gPDY}/${gcyc}/${COMPONENT}" -export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${gPDY}/${gcyc}/${COMPONENT}" - -# Add UFSDA to PYTHONPATH -export PYTHONPATH=${HOMEgfs}/sorc/gdas.cd/ush/:${PYTHONPATH} +export COMIN_GES="${ROTDIR}/${GDUMP}.${gPDY}/${gcyc}/atmos" +export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${gPDY}/${gcyc}/atmos" ############################################################### # Run relevant script -EXSCRIPT=${GDASRUNSH:-${HOMEgfs}/scripts/exgdas_global_atmos_ensanal_run.sh} +EXSCRIPT=${GDASATMENSFINALPY:-${HOMEgfs}/scripts/exglobal_atmens_analysis_finalize.py} ${EXSCRIPT} status=$? -[[ ${status} -ne 0 ]] && exit ${status} +[[ ${status} -ne 0 ]] && exit "${status}" ############################################## # End JOB SPECIFIC work @@ -53,14 +49,14 @@ status=$? ############################################## # Final processing ############################################## -if [ -e "${pgmout}" ] ; then - cat ${pgmout} +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" fi ########################################## # Remove the Temporary working directory ########################################## -cd ${DATAROOT} -[[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} +cd "${DATAROOT}" || exit 1 +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" exit 0 diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE new file mode 100755 index 00000000000..8dcd0e9fb91 --- /dev/null +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE @@ -0,0 +1,55 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +export DATA=${DATA:-${DATAROOT}/${RUN}atmensanl_${cyc}} +source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlinit" -c "base atmensanl atmensanlinit" + +############################################## +# Set variables used in the script +############################################## +export CDATE=${CDATE:-${PDY}${cyc}} +export CDUMP=${CDUMP:-${RUN:-"gfs"}} + + +############################################## +# Begin JOB SPECIFIC work +############################################## + +GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") +export GDATE +export gPDY=${GDATE:0:8} +export gcyc=${GDATE:8:2} +export GDUMP=${GDUMP:-"gdas"} + +export OPREFIX="${CDUMP_OBS}.t${cyc}z." +export GPREFIX="${GDUMP}.t${gcyc}z." +export APREFIX="${CDUMP}.t${cyc}z." + +export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/atmos} + +mkdir -p "${COMOUT}" + +# COMIN_GES and COMIN_GES_ENS are used in script +export COMIN_GES="${ROTDIR}/${GDUMP}.${gPDY}/${gcyc}/atmos" +export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${gPDY}/${gcyc}" + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASATMENSINITPY:-${HOMEgfs}/scripts/exglobal_atmens_analysis_initialize.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN b/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN new file mode 100755 index 00000000000..a8dbf4aeb41 --- /dev/null +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN @@ -0,0 +1,56 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +export WIPE_DATA="NO" +export DATA=${DATA:-${DATAROOT}/${RUN}atmensanl_${cyc}} +source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlrun" -c "base atmensanl atmensanlrun" + +############################################## +# Set variables used in the script +############################################## +export CDATE=${CDATE:-${PDY}${cyc}} +export CDUMP=${CDUMP:-${RUN:-"gfs"}} + + +############################################## +# Begin JOB SPECIFIC work +############################################## + +GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") +export GDATE +gPDY=${GDATE:0:8} +export gcyc=${GDATE:8:2} +export GDUMP=${GDUMP:-"gdas"} + +export OPREFIX="${CDUMP}.t${cyc}z." +export GPREFIX="${GDUMP}.t${gcyc}z." +export APREFIX="${CDUMP}.t${cyc}z." + +export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/atmos} + +mkdir -p "${COMOUT}" + +# COMIN_GES and COMIN_GES_ENS are used in script +export COMIN_GES="${ROTDIR}/${GDUMP}.${gPDY}/${gcyc}/atmos" +export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${gPDY}/${gcyc}/atmos" + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASATMENSRUNSH:-${HOMEgfs}/scripts/exglobal_atmens_analysis_run.sh} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/rocoto/atmensanalpost.sh b/jobs/rocoto/atmensanalpost.sh deleted file mode 100755 index 91ac2d62123..00000000000 --- a/jobs/rocoto/atmensanalpost.sh +++ /dev/null @@ -1,20 +0,0 @@ -#! /usr/bin/env bash - -export STRICT="NO" -source "${HOMEgfs}/ush/preamble.sh" -export STRICT="YES" - -############################################################### -# Source UFSDA workflow modules -. ${HOMEgfs}/ush/load_ufsda_modules.sh -status=$? -[[ ${status} -ne 0 ]] && exit ${status} - -export job="atmensanalpost" -export jobid="${job}.$$" - -############################################################### -# Execute the JJOB -${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST -status=$? -exit ${status} diff --git a/jobs/rocoto/atmensanalprep.sh b/jobs/rocoto/atmensanalprep.sh deleted file mode 100755 index b54a1b464ed..00000000000 --- a/jobs/rocoto/atmensanalprep.sh +++ /dev/null @@ -1,20 +0,0 @@ -#! /usr/bin/env bash - -export STRICT="NO" -source "${HOMEgfs}/ush/preamble.sh" -export STRICT="YES" - -############################################################### -# Source UFSDA workflow modules -. ${HOMEgfs}/ush/load_ufsda_modules.sh -status=$? -[[ ${status} -ne 0 ]] && exit ${status} - -export job="atmensanalprep" -export jobid="${job}.$$" - -############################################################### -# Execute the JJOB -${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP -status=$? -exit ${status} diff --git a/jobs/rocoto/atmensanlfinal.sh b/jobs/rocoto/atmensanlfinal.sh new file mode 100755 index 00000000000..838e9712f87 --- /dev/null +++ b/jobs/rocoto/atmensanlfinal.sh @@ -0,0 +1,23 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="atmensanlfinal" +export jobid="${job}.$$" + +############################################################### +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE" +status=$? +exit "${status}" diff --git a/jobs/rocoto/atmensanlinit.sh b/jobs/rocoto/atmensanlinit.sh new file mode 100755 index 00000000000..0ab78a1083a --- /dev/null +++ b/jobs/rocoto/atmensanlinit.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="atmensanlinit" +export jobid="${job}.$$" + +############################################################### +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE" +status=$? +exit "${status}" diff --git a/jobs/rocoto/atmensanalrun.sh b/jobs/rocoto/atmensanlrun.sh similarity index 57% rename from jobs/rocoto/atmensanalrun.sh rename to jobs/rocoto/atmensanlrun.sh index a2509a310ea..8bfbc7f6629 100755 --- a/jobs/rocoto/atmensanalrun.sh +++ b/jobs/rocoto/atmensanlrun.sh @@ -1,20 +1,18 @@ #! /usr/bin/env bash -export STRICT="NO" source "${HOMEgfs}/ush/preamble.sh" -export STRICT="YES" ############################################################### # Source UFSDA workflow modules -. ${HOMEgfs}/ush/load_ufsda_modules.sh +. "${HOMEgfs}/ush/load_ufsda_modules.sh" status=$? -[[ ${status} -ne 0 ]] && exit ${status} +[[ ${status} -ne 0 ]] && exit "${status}" -export job="atmensanalrun" +export job="atmensanlrun" export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN +"${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN" status=$? -exit ${status} +exit "${status}" diff --git a/parm/config/config.atmensanal b/parm/config/config.atmensanal deleted file mode 100644 index 2c939f0d84e..00000000000 --- a/parm/config/config.atmensanal +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -x - -########## config.atmensanal ########## -# configuration common to all atm atmensanal analysis tasks - -echo "BEGIN: config.atmensanal" - -export OBS_YAML_DIR=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/config/ -export OBS_LIST=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/lists/lgetkf_prototype.yaml -export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/atm/berror/hybvar_bump.yaml -export ATMENSYAML=$HOMEgfs/sorc/gdas.cd/parm/atm/lgetkf/lgetkf.yaml -export FV3JEDI_FIX=$HOMEgfs/fix/gdas -export R2D2_OBS_DB='ufsda_test' -export R2D2_OBS_DUMP='oper_gdas' -export R2D2_OBS_SRC='ncdiag' -export R2D2_BC_SRC='gsi' -export R2D2_BC_DUMP='oper_gdas' -export R2D2_ARCH_DB='local' -export INTERP_METHOD='barycentric' - -export io_layout_x=1 # hardwired to 1,1 in yamltools.py -export io_layout_y=1 - -echo "END: config.atmensanal" diff --git a/parm/config/config.atmensanalpost b/parm/config/config.atmensanalpost deleted file mode 100644 index f79ee5b5078..00000000000 --- a/parm/config/config.atmensanalpost +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -x - -########## config.atmensanalpost ########## -# Post Atm Analysis specific - -echo "BEGIN: config.atmensanalpost" - -# Get task specific resources -. $EXPDIR/config.resources atmensanalpost -echo "END: config.atmensanalpost" diff --git a/parm/config/config.atmensanalprep b/parm/config/config.atmensanalprep deleted file mode 100644 index b719b9ac6cd..00000000000 --- a/parm/config/config.atmensanalprep +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -x - -########## config.atmensanalprep ########## -# Pre Atm Analysis specific - -echo "BEGIN: config.atmensanalprep" - -# Get task specific resources -. $EXPDIR/config.resources atmensanalprep -echo "END: config.atmensanalprep" diff --git a/parm/config/config.atmensanalrun b/parm/config/config.atmensanalrun deleted file mode 100644 index aeb59d18059..00000000000 --- a/parm/config/config.atmensanalrun +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -x - -########## config.atmensanalrun ########## -# Atm LETKFs specific - -echo "BEGIN: config.atmensanalrun" - -# Get task specific resources -. $EXPDIR/config.resources atmensanalrun - -# Task specific variables -export JEDIENSEXE=$HOMEgfs/exec/fv3jedi_letkf.x - -echo "END: config.atmensanalrun" diff --git a/parm/config/config.atmensanl b/parm/config/config.atmensanl new file mode 100755 index 00000000000..59454db4b08 --- /dev/null +++ b/parm/config/config.atmensanl @@ -0,0 +1,23 @@ +#!/bin/bash -x + +########## config.atmensanl ########## +# configuration common to all atm ens analysis tasks + +echo "BEGIN: config.atmensanl" + +export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/atm/obs/config/ +export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/atm/obs/lists/lgetkf_prototype.yaml +export ATMENSYAML=${HOMEgfs}/sorc/gdas.cd/parm/atm/lgetkf/lgetkf.yaml +export FV3JEDI_FIX=${HOMEgfs}/fix/gdas +export INTERP_METHOD='barycentric' + +export layout_x=1 +export layout_y=1 + +export io_layout_x=1 +export io_layout_y=1 + +export JEDIENSEXE=${HOMEgfs}/exec/fv3jedi_letkf.x +export crtm_VERSION="2.3.0" + +echo "END: config.atmensanl" diff --git a/parm/config/config.atmensanlfinal b/parm/config/config.atmensanlfinal new file mode 100755 index 00000000000..db3b8b69c97 --- /dev/null +++ b/parm/config/config.atmensanlfinal @@ -0,0 +1,10 @@ +#!/bin/bash -x + +########## config.atmensanlfinal ########## +# Post Atm Ens Analysis specific + +echo "BEGIN: config.atmensanlfinal" + +# Get task specific resources +. "${EXPDIR}/config.resources" atmensanlfinal +echo "END: config.atmensanlfinal" diff --git a/parm/config/config.atmensanlinit b/parm/config/config.atmensanlinit new file mode 100755 index 00000000000..a9cd4c968df --- /dev/null +++ b/parm/config/config.atmensanlinit @@ -0,0 +1,10 @@ +#!/bin/bash -x + +########## config.atmensanlinit ########## +# Pre Atm Ens Analysis specific + +echo "BEGIN: config.atmensanlinit" + +# Get task specific resources +. "${EXPDIR}/config.resources" atmensanlinit +echo "END: config.atmensanlinit" diff --git a/parm/config/config.atmensanlrun b/parm/config/config.atmensanlrun new file mode 100755 index 00000000000..aec8e04ec03 --- /dev/null +++ b/parm/config/config.atmensanlrun @@ -0,0 +1,11 @@ +#!/bin/bash -x + +########## config.atmensanlrun ########## +# Atm Ens Analysis specific + +echo "BEGIN: config.atmensanlrun" + +# Get task specific resources +. "${EXPDIR}/config.resources" atmensanlrun + +echo "END: config.atmensanlrun" diff --git a/parm/config/config.resources b/parm/config/config.resources index e821a148cb3..934999e1de1 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -10,7 +10,7 @@ if [ $# -ne 1 ]; then echo "argument can be any one of the following:" echo "getic init coupled_ic aerosol_init" echo "atmanalprep atmanalrun atmanalpost" - echo "atmensanalprep atmensanalrun atmensanalpost" + echo "atmensanlinit atmensanlrun atmensanlfinal" echo "aeroanlinit aeroanlrun aeroanlfinal" echo "anal sfcanl analcalc analdiag gldas fcst post vrfy metp arch echgres" echo "eobs ediag eomg eupd ecen esfc efcs epos earc" @@ -662,34 +662,39 @@ elif [ ${step} = "coupled_ic" ]; then export nth_coupled_ic=1 export is_exclusive=True -elif [ ${step} = "atmensanalprep" ]; then +elif [[ "${step}" = "atmensanlinit" ]]; then - export wtime_atmensanalprep="00:10:00" - export npe_atmensanalprep=1 - export nth_atmensanalprep=1 - export npe_node_atmensanalprep=$(echo "${npe_node_max} / ${nth_atmensanalprep}" | bc) - export is_exclusive=True + export wtime_atmensanlinit="00:10:00" + export npe_atmensanlinit=1 + export nth_atmensanlinit=1 + npe_node_atmensanlinit=$(echo "${npe_node_max} / ${nth_atmensanlinit}" | bc) + export npe_node_atmensanlinit + export memory_atmensanlinit="3072M" -elif [ ${step} = "atmensanalrun" ]; then +elif [[ "${step}" = "atmensanlrun" ]]; then # make below case dependent later - export layout_x=2 - export layout_y=3 - - export wtime_atmensanalrun="00:30:00" - export npe_atmensanalrun=$(echo "${layout_x} * ${layout_y} * 6" | bc) - export npe_atmensanalrun_gfs=$(echo "${layout_x} * ${layout_y} * 6" | bc) - export nth_atmensanalrun=1 - export nth_atmensanalrun_gfs=${nth_atmensanalrun} + export layout_x=1 + export layout_y=1 + + export wtime_atmensanlrun="00:30:00" + npe_atmensanlrun=$(echo "${layout_x} * ${layout_y} * 6" | bc) + export npe_atmensanlrun + npe_atmensanlrun_gfs=$(echo "${layout_x} * ${layout_y} * 6" | bc) + export npe_atmensanlrun_gfs + export nth_atmensanlrun=1 + export nth_atmensanlrun_gfs=${nth_atmensanlrun} + npe_node_atmensanlrun=$(echo "${npe_node_max} / ${nth_atmensanlrun}" | bc) + export npe_node_atmensanlrun export is_exclusive=True - export npe_node_atmensanalrun=$(echo "${npe_node_max} / ${nth_atmensanalrun}" | bc) -elif [ ${step} = "atmensanalpost" ]; then +elif [[ "${step}" = "atmensanlfinal" ]]; then - export wtime_atmensanalpost="00:30:00" - export npe_atmensanalpost=${npe_node_max} - export nth_atmensanalpost=1 - export npe_node_atmensanalpost=$(echo "${npe_node_max} / ${nth_atmensanalpost}" | bc) + export wtime_atmensanlfinal="00:30:00" + export npe_atmensanlfinal=${npe_node_max} + export nth_atmensanlfinal=1 + npe_node_atmensanlfinal=$(echo "${npe_node_max} / ${nth_atmensanlfinal}" | bc) + export npe_node_atmensanlfinal export is_exclusive=True elif [[ ${step} = "eobs" || ${step} = "eomg" ]]; then diff --git a/scripts/exgdas_global_atmos_ensanal_post.py b/scripts/exgdas_global_atmos_ensanal_post.py deleted file mode 100755 index 6c5384953f7..00000000000 --- a/scripts/exgdas_global_atmos_ensanal_post.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 -################################################################################ -# UNIX Script Documentation Block -# . . -# Script name: exgdas_global_atmos_analysis_post.py -# Script description: Post atmospheric analysis script. -# -# Author: Cory Martin Org: NCEP/EMC Date: 2021-12-29 -# -# Abstract: This script runs after the atmospheric analysis and -# archives each diagnostic file into the R2D2 local user database. -# -# $Id$ -# -# Attributes: -# Language: Python3 -# -################################################################################ - -# import os and sys to add ush to path -import logging -import os -import sys - -# set up logger -logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') - -# get absolute path of ush/ directory either from env or relative to this file -my_dir = os.path.dirname(__file__) -my_home = os.path.dirname(os.path.dirname(my_dir)) -gdas_home = os.path.join(os.environ['HOMEgfs'], 'sorc', 'gdas.cd') -sys.path.append(os.path.join(os.getenv('HOMEgfs', my_home), 'ush')) -logging.info(f"sys.path={sys.path}") - -# import UFSDA utilities -import ufsda - -# get configuration based on environment variables -config = ufsda.misc_utils.get_env_config(component='atm') -config['DIAG_DIR'] = os.path.join(os.environ['COMOUT_ENS'], 'diags') -config['provider'] = 'ncdiag_lgetkf' - -# use R2D2 to archive hofx files -ufsda.archive.atm_diags(config) diff --git a/scripts/exgdas_global_atmos_ensanal_run.sh b/scripts/exgdas_global_atmos_ensanal_run.sh deleted file mode 100755 index 2e87573eda1..00000000000 --- a/scripts/exgdas_global_atmos_ensanal_run.sh +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/bash -################################################################################ -#### UNIX Script Documentation Block -# . . -# Script name: exgdas_global_atmos_analysis_run.sh -# Script description: Runs the global atmospheric analysis with FV3-JEDI -# -# Author: Cory Martin Org: NCEP/EMC Date: 2021-12-28 -# -# Abstract: This script makes a global model atmospheric analysis using FV3-JEDI -# and also (for now) updates increment files using a python ush utility -# -# $Id$ -# -# Attributes: -# Language: POSIX shell -# Machine: Orion -# -################################################################################ - -# Set environment. -source "$HOMEgfs/ush/preamble.sh" - -# Directories -pwd=$(pwd) - -# Utilities -export NLN=${NLN:-"/bin/ln -sf"} -export INCPY=${INCPY:-"$HOMEgfs/sorc/gdas.cd/ush/jediinc2fv3.py"} -export GENYAML=${GENYAML:-"$HOMEgfs/sorc/gdas.cd/ush/genYAML"} -export GETOBSYAML=${GETOBSYAML:-"$HOMEgfs/sorc/gdas.cd/ush/get_obs_list.py"} - -################################################################################ -# make subdirectories -mkdir -p $DATA/fv3jedi -mkdir -p $DATA/obs -mkdir -p $DATA/diags -mkdir -p $DATA/bc -mkdir -p $DATA/anl - -################################################################################ -# generate YAML file -cat > $DATA/temp.yaml << EOF -template: ${ATMENSYAML} -output: $DATA/fv3jedi_ens.yaml -config: - atm: true - BERROR_YAML: $BERROR_YAML - OBS_DIR: obs - DIAG_DIR: diags - CRTM_COEFF_DIR: crtm - BIAS_IN_DIR: obs - BIAS_OUT_DIR: bc - OBS_PREFIX: $OPREFIX - BIAS_PREFIX: $GPREFIX - OBS_LIST: $OBS_LIST - OBS_YAML_DIR: $OBS_YAML_DIR - BKG_DIR: bkg - fv3jedi_staticb_dir: berror - fv3jedi_fix_dir: fv3jedi - fv3jedi_fieldset_dir: fv3jedi - fv3jedi_fieldmetadata_dir: fv3jedi - OBS_DATE: '$CDATE' - BIAS_DATE: '$GDATE' - ANL_DIR: anl/ - NMEM_ENKF: '$NMEM_ENKF' - INTERP_METHOD: '$INTERP_METHOD' -EOF -$GENYAML --config $DATA/temp.yaml - -################################################################################ -# link observations to $DATA -$GETOBSYAML --config $DATA/fv3jedi_ens.yaml --output $DATA/${OPREFIX}obsspace_list -files=$(cat $DATA/${OPREFIX}obsspace_list) -for file in $files; do - basefile=$(basename $file) - $NLN $COMIN/$basefile $DATA/obs/$basefile -done - -# link backgrounds to $DATA -# linking FMS RESTART files for now -# change to (or make optional) for cube sphere history later -##$NLN ${COMIN_GES}/RESTART $DATA/bkg - - -# Link ensemble backgrounds to $DATA. Make directories -# for ensemble output -if [ $DOHYBVAR = "YES" -o $DO_JEDIENS = "YES" ]; then - mkdir -p $DATA/bkg - for imem in $(seq 1 $NMEM_ENKF); do - memchar="mem"$(printf %03i $imem) - mkdir -p $DATA/bkg/$memchar - $NLN ${COMIN_GES_ENS}/$memchar/RESTART $DATA/bkg/$memchar - mkdir -p $DATA/anl/$memchar - done -fi - -################################################################################ -# link fix files to $DATA -# static B -##CASE_BERROR=${CASE_BERROR:-${CASE_ANL:-$CASE}} -##$NLN $FV3JEDI_FIX/bump/$CASE_BERROR/ $DATA/berror - -# vertical coordinate -LAYERS=$(expr $LEVS - 1) -$NLN $FV3JEDI_FIX/fv3jedi/fv3files/akbk${LAYERS}.nc4 $DATA/fv3jedi/akbk.nc4 - -# other FV3-JEDI fix files -$NLN $FV3JEDI_FIX/fv3jedi/fv3files/fmsmpp.nml $DATA/fv3jedi/fmsmpp.nml -$NLN $FV3JEDI_FIX/fv3jedi/fv3files/field_table_gfdl $DATA/fv3jedi/field_table - -# fieldmetadata -$NLN $FV3JEDI_FIX/fv3jedi/fieldmetadata/gfs-restart.yaml $DATA/fv3jedi/gfs-restart.yaml - -# fieldsets -fieldsets="dynamics.yaml ufo.yaml" -for fieldset in $fieldsets; do - $NLN $FV3JEDI_FIX/fv3jedi/fieldsets/$fieldset $DATA/fv3jedi/$fieldset -done - -# CRTM coeffs -${NLN} "${FV3JEDI_FIX}/crtm/2.3.0" "${DATA}/crtm" - -# Link executable to $DATA -$NLN $JEDIENSEXE $DATA/fv3jedi_ens.x - -################################################################################ -# run executable -export OMP_NUM_THREADS=$NTHREADS_ATMENSANAL -export pgm=$JEDIENSEXE -. prep_step -$APRUN_ATMENSANAL $DATA/fv3jedi_ens.x $DATA/fv3jedi_ens.yaml 1>&1 2>&2 -export err=$?; err_chk - -################################################################################ -# translate FV3-JEDI increment to FV3 readable format -for imem in $(seq 1 $NMEM_ENKF); do - memchar="mem"$(printf %03i $imem) - atmges_fv3=$COMIN_GES_ENS/$memchar/${GPREFIX}atmf006.nc - atminc_jedi=$DATA/anl/$memchar/atminc.${PDY}_${cyc}0000z.nc4 - atminc_fv3=$COMOUT_ENS/$memchar/${CDUMP}.${cycle}.atminc.nc - mkdir -p $COMOUT_ENS/$memchar - if [ -s $atminc_jedi ]; then - $INCPY $atmges_fv3 $atminc_jedi $atminc_fv3 - export err=$? - else - echo "***WARNING*** missing $atminc_jedi ABORT" - export err=99 - fi - err_chk -done - -################################################################################ -# Create log file noting creating of analysis increment file -echo "$CDUMP $CDATE atminc done at $(date)" > $COMOUT_ENS/${CDUMP}.${cycle}.loginc.txt - -################################################################################ -# Copy diags and YAML to $COMOUT -cp -r ${DATA}/fv3jedi_ens.yaml ${COMOUT_ENS}/${CDUMP}.${cycle}.fv3jedi_ens.yaml -cp -rf "${DATA}/diags" "${COMOUT_ENS}/" - - -################################################################################ - -exit ${err} - -################################################################################ diff --git a/scripts/exglobal_atmens_analysis_finalize.py b/scripts/exglobal_atmens_analysis_finalize.py new file mode 100755 index 00000000000..eb41b180711 --- /dev/null +++ b/scripts/exglobal_atmens_analysis_finalize.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# exgdas_global_atmens_analysis_finalize.py +# This script creates an AtmEnsAnalysis class +# and runs the finalize method +# which perform post-processing and clean up activities +# for a global atmens variational analysis +import os + +from pygw.logger import Logger, logit +from pygw.configuration import cast_strdict_as_dtypedict +from pygfs.task.atmens_analysis import AtmEnsAnalysis + + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the atmens analysis task + AtmEnsAnl = AtmEnsAnalysis(config) + AtmEnsAnl.finalize() diff --git a/scripts/exglobal_atmens_analysis_initialize.py b/scripts/exglobal_atmens_analysis_initialize.py new file mode 100755 index 00000000000..f2776190c03 --- /dev/null +++ b/scripts/exglobal_atmens_analysis_initialize.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# exgdas_global_atmens_analysis_initialize.py +# This script creates an AtmEnsAnalysis class +# and runs the initialize method +# which create and stage the runtime directory +# and create the YAML configuration +# for a global atmens variational analysis +import os + +from pygw.logger import Logger +from pygw.configuration import cast_strdict_as_dtypedict +from pygfs.task.atmens_analysis import AtmEnsAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the atmens analysis task + AtmEnsAnl = AtmEnsAnalysis(config) + AtmEnsAnl.initialize() diff --git a/scripts/exglobal_atmens_analysis_run.sh b/scripts/exglobal_atmens_analysis_run.sh new file mode 100755 index 00000000000..19ce9017102 --- /dev/null +++ b/scripts/exglobal_atmens_analysis_run.sh @@ -0,0 +1,16 @@ +#!/bin/bash +################################################################################ +# exgdas_global_atmens_analysis_run.sh +# +# This script runs a global atmens variational analysis with FV3-JEDI. +# It assumes the runtime directory has already been staged with the appropriate +# input files and YAML configuration (by the initialize script) before execution. +# +################################################################################ +# run executable +set -x +export pgm=${JEDIVAREXE} +. prep_step +${APRUN_ATMENSANL} "${DATA}/fv3jedi_letkf.x" "${DATA}/${CDUMP}.t${cyc}z.atmens.yaml" 1>&1 2>&2 +export err=$?; err_chk +exit "${err}" diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py new file mode 100644 index 00000000000..a261bd3543a --- /dev/null +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 + +import os +import glob +import gzip +import tarfile +from netCDF4 import Dataset +from logging import getLogger +from typing import Dict, List, Any + +from pygw.attrdict import AttrDict +from pygw.file_utils import FileHandler +from pygw.timetools import to_isotime, to_fv3time, to_timedelta +from pygw.fsutils import rm_p +from pygw.template import Template, TemplateConstants +from pygw.yaml_file import YAMLFile +from pygw.logger import logit +from pygfs.task.analysis import Analysis + +logger = getLogger(__name__.split('.')[-1]) + + +class AtmEnsAnalysis(Analysis): + """ + Class for global atmens analysis tasks + """ + @logit(logger, name="AtmEnsAnalysis") + def __init__(self, config): + super().__init__(config) + + _res = int(self.config['CASE_ANL'][1:]) + _res_enkf = int(self.config['CASE_ENKF'][1:]) + _window_begin = self.runtime_config.current_cycle - to_timedelta(f"{self.config['assim_freq']}H") / 2 + + # Create a local dictionary that is repeatedly used across this class + local_dict = AttrDict( + { + 'npx_ges': _res + 1, + 'npy_ges': _res + 1, + 'npz_ges': self.config.LEVS - 1, + 'npz': self.config.LEVS - 1, + 'npx_anl': _res_enkf + 1, + 'npy_anl': _res_enkf + 1, + 'npz_anl': self.config['LEVS'] - 1, + 'ATM_WINDOW_BEGIN': to_isotime(_window_begin), + 'ATM_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H", + 'BKG_ISOTIME': to_isotime(self.runtime_config.current_cycle), + 'BKG_YYYYmmddHHMMSS': to_fv3time(self.runtime_config.current_cycle), + 'cdate_fv3': to_fv3time(self.runtime_config.current_cycle), + 'comin_ges_atm': self.config.COMIN_GES, + 'comin_ges_atmens': self.config.COMIN_GES_ENS, + } + ) + + # task_config is everything that this task should need + self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) + + @logit(logger) + def initialize(self: Analysis) -> None: + """Initialize a global atmens analysis + + This method will initialize a global atmens analysis using JEDI. + This includes: + - staging CRTM fix files + - staging FV3-JEDI fix files + - staging B error files + - staging model backgrounds + - generating a YAML file for the JEDI executable + - linking the JEDI executable (TODO make it copyable, requires JEDI fix) + - creating output directories + """ + + # stage observations and bias corrections + super().initialize() + + # stage CRTM fix files + super().stage_fix('atm_crtm_coeff.yaml') + + # stage fix files + super().stage_fix('atm_jedi_fix.yaml') + + # stage backgrounds + FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() + + # generate variational YAML file + yaml_out = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.atmens.yaml") + varda_yaml = YAMLFile(path=self.task_config['ATMENSYAML']) + varda_yaml = Template.substitute_structure(varda_yaml, TemplateConstants.DOUBLE_CURLY_BRACES, self.task_config.get) + varda_yaml = Template.substitute_structure(varda_yaml, TemplateConstants.DOLLAR_PARENTHESES, self.task_config.get) + varda_yaml.save(yaml_out) + logger.info(f"Wrote YAML to {yaml_out}") + + # link var executable + exe_src = self.task_config['JEDIENSEXE'] + exe_dest = os.path.join(self.task_config['DATA'], os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + # need output dir for diags and anl + newdirs = [ + os.path.join(self.task_config['DATA'], 'anl'), + os.path.join(self.task_config['DATA'], 'diags'), + ] + FileHandler({'mkdir': newdirs}).sync() + + # Make directories for member analsis files + for imem in range(1, self.task_config['NMEM_ENKF'] + 1): + memchar = f"mem{imem:03d}" + anldir = [ + os.path.join(self.task_config['DATA'], 'anl', memchar) + ] + FileHandler({'mkdir': anldir}).sync() + + @logit(logger) + def finalize(self: Analysis) -> None: + """Finalize a global atmens analysis + + This method will finalize a global atmens analysis using JEDI. + This includes: + - tarring up output diag files and place in ROTDIR + - copying the generated YAML file from initialize to the ROTDIR + - copying the guess files to the ROTDIR + - applying the increments to the original RESTART files + - moving the increment files to the ROTDIR + + Please note that some of these steps are temporary and will be modified + once the model is able to read atmens tracer increments. + """ + # ---- tar up diags + # path of output tar statfile + atmensstat = os.path.join(self.task_config['COMOUT'], f"{self.task_config['APREFIX']}atmensstat") + + # get list of diag files to put in tarball + diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc4')) + + # gzip the files first + for diagfile in diags: + with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: + f_out.writelines(f_in) + + # open tar file for writing + with tarfile.open(atmensstat, "w") as archive: + for diagfile in diags: + archive.add(f"{diagfile}.gz") + + # copy full YAML from executable to ROTDIR + src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.atmens.yaml") + dest = os.path.join(self.task_config['COMOUT'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.atmens.yaml") + yaml_copy = { + 'mkdir': [self.task_config['COMOUT']], + 'copy': [[src, dest]] + } + FileHandler(yaml_copy).sync() + + # ---- NOTE below is 'temporary', eventually we will not be using FMS RESTART formatted files + # ---- all of the rest of this method will need to be changed but requires model and JEDI changes + # ---- copy RESTART fv_tracer files for future reference + # fms_bkg_file_template = os.path.join(self.task_config.comin_ges_atmens, 'RESTART', f'{self.task_config.cdate_fv3}.fv_tracer.res.tileX.nc') + # bkglist = [] + # for itile in range(1, self.task_config.ntiles + 1): + # bkg_path = fms_bkg_file_template.replace('tileX', f'tile{itile}') + # dest = os.path.join(self.task_config['COMOUT'], f'atmges.{os.path.basename(bkg_path)}') + # bkglist.append([bkg_path, dest]) + # FileHandler({'copy': bkglist}).sync() + + # ---- add increments to RESTART files + # logger.info('Adding increments to RESTART files') + # self._add_fms_cube_sphere_increments() + + # ---- move increments to ROTDIR + # logger.info('Moving increments to ROTDIR') + # fms_inc_file_template = os.path.join(self.task_config['DATA'], 'anl', f'atminc.{self.task_config.cdate_fv3}.fv_tracer.res.tileX.nc') + # inclist = [] + # for itile in range(1, self.task_config.ntiles + 1): + # inc_path = fms_inc_file_template.replace('tileX', f'tile{itile}') + # dest = os.path.join(self.task_config['COMOUT'], os.path.basename(inc_path)) + # inclist.append([inc_path, dest]) + # FileHandler({'copy': inclist}).sync() + + # ---- copy member increments to ROTDIR + cdate_inc = self.task_config.cdate_fv3.replace('.', '_') + inclist = [] + for imem in range(1, self.task_config['NMEM_ENKF'] + 1): + memchar = f"mem{imem:03d}" + + # make directory for member incrfement + incdir = [ + os.path.join(self.task_config['COMOUT'], memchar, 'atmos') + ] + FileHandler({'mkdir': incdir}).sync() + + src = os.path.join(self.task_config['DATA'], 'anl', memchar, f'atminc.{cdate_inc}z.nc4') + dest = os.path.join(self.task_config['COMOUT'], memchar, 'atmos', f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.atminc.nc") + cdate_inc = self.task_config.cdate_fv3.replace('.', '_') + inclist.append([src, dest]) + + inc_dict = { + 'copy': inclist, + } + FileHandler(inc_dict).sync() + + def clean(self): + super().clean() + + @logit(logger) + def _add_fms_cube_sphere_increments(self: Analysis) -> None: + """This method adds increments to RESTART files to get an analysis + NOTE this is only needed for now because the model cannot read atmens increments. + This method will be assumed to be deprecated before this is implemented operationally + """ + # only need the fv_tracer files + fms_inc_file_template = os.path.join(self.task_config['DATA'], 'anl', f'atminc.{self.task_config.cdate_fv3}.fv_tracer.res.tileX.nc') + fms_bkg_file_template = os.path.join(self.task_config.comin_ges_atm, 'RESTART', f'{self.task_config.cdate_fv3}.fv_tracer.res.tileX.nc') + # get list of increment vars + incvars_list_path = os.path.join(self.task_config['HOMEgfs'], 'parm', 'parm_gdas', 'atmensanl_inc_vars.yaml') + incvars = YAMLFile(path=incvars_list_path) + super().add_fv3_increments(fms_inc_file_template, fms_bkg_file_template, incvars) + + @logit(logger) + def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: + """Compile a dictionary of model background files to copy + + This method constructs a dictionary of FV3 RESTART files (coupler, core, tracer) + that are needed for global atmens DA and returns said dictionary for use by the FileHandler class. + + Parameters + ---------- + task_config: Dict + a dictionary containing all of the configuration needed for the task + + Returns + ---------- + bkg_dict: Dict + a dictionary containing the list of model background files to copy for FileHandler + """ + super().get_bkg_dict(task_config) + # NOTE for now this is FV3 RESTART files and just assumed to be fh006 + + bkgdir = [ + os.path.join(task_config['DATA'], 'bkg'), + ] + FileHandler({'mkdir': bkgdir}).sync() + + # loop over ensemble members + bkglist = [] + for imem in range(1, task_config['NMEM_ENKF'] + 1): + memchar = f"mem{imem:03d}" + + # make run directory for member restart files + bkgdir = [ + os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART') + ] + FileHandler({'mkdir': bkgdir}).sync() + + # get FV3 RESTART files, this will be a lot simpler when using history files + rst_dir = os.path.join(task_config.comin_ges_atmens, memchar, 'atmos/RESTART') + + basename = f'{task_config.cdate_fv3}.coupler.res' + bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) + basename = f'{task_config.cdate_fv3}.fv_core.res.nc' + bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) + basename_cadat = f'{task_config.cdate_fv3}.ca_data.tileX.nc' + basename_core = f'{task_config.cdate_fv3}.fv_core.res.tileX.nc' + basename_srfwnd = f'{task_config.cdate_fv3}.fv_srf_wnd.res.tileX.nc' + basename_tracer = f'{task_config.cdate_fv3}.fv_tracer.res.tileX.nc' + basename_phydat = f'{task_config.cdate_fv3}.phy_data.tileX.nc' + basename_sfcdat = f'{task_config.cdate_fv3}.sfc_data.tileX.nc' + for itile in range(1, task_config.ntiles + 1): + basename = basename_cadat.replace('tileX', f'tile{itile}') + bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) + basename = basename_core.replace('tileX', f'tile{itile}') + bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) + basename = basename_srfwnd.replace('tileX', f'tile{itile}') + bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) + basename = basename_tracer.replace('tileX', f'tile{itile}') + bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) + basename = basename_phydat.replace('tileX', f'tile{itile}') + bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) + basename = basename_sfcdat.replace('tileX', f'tile{itile}') + bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) + + bkg_dict = { + 'copy': bkglist, + } + return bkg_dict diff --git a/workflow/applications.py b/workflow/applications.py index 717940a1bda..4994b0e0245 100644 --- a/workflow/applications.py +++ b/workflow/applications.py @@ -107,7 +107,7 @@ def __init__(self, conf: Configuration) -> None: self.do_vrfy = _base.get('DO_VRFY', True) self.do_metp = _base.get('DO_METP', False) self.do_jedivar = _base.get('DO_JEDIVAR', False) - self.do_jediens = _base.get('DO_JEDIENS', False) + self.do_jediatmens = _base.get('DO_JEDIENS', False) self.do_jediocnvar = _base.get('DO_JEDIOCNVAR', False) self.do_hpssarch = _base.get('HPSSARCH', False) @@ -192,8 +192,8 @@ def _cycled_configs(self): configs += ['gldas'] if self.do_hybvar: - if self.do_jediens: - configs += ['atmensanalprep', 'atmensanalrun', 'atmensanalpost'] + if self.do_jediatmens: + configs += ['atmensanlinit', 'atmensanlrun', 'atmensanlfinal'] else: configs += ['eobs', 'eomg', 'ediag', 'eupd'] configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] @@ -374,8 +374,8 @@ def _get_cycled_task_names(self): hybrid_tasks = [] hybrid_after_eupd_tasks = [] if self.do_hybvar: - if self.do_jediens: - hybrid_tasks += ['atmensanalprep', 'atmensanalrun', 'atmensanalpost', 'echgres'] + if self.do_jediatmens: + hybrid_tasks += ['atmensanlinit', 'atmensanlrun', 'atmensanlfinal', 'echgres'] else: hybrid_tasks += ['eobs', 'eupd', 'echgres'] hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py index 48b03254763..b2e22dd074b 100644 --- a/workflow/rocoto/workflow_tasks.py +++ b/workflow/rocoto/workflow_tasks.py @@ -16,7 +16,7 @@ class Tasks: 'ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalpost', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', - 'atmensanalprep', 'atmensanalrun', 'atmensanalpost', + 'atmensanlprep', 'atmensanlrun', 'atmensanlpost', 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', 'fcst', 'post', 'ocnpost', 'vrfy', 'metp', 'postsnd', 'awips', 'gempak', @@ -1101,57 +1101,44 @@ def eupd(self): return task - def atmensanalprep(self): - - dump_suffix = self._base["DUMP_SUFFIX"] - gfs_cyc = self._base["gfs_cyc"] - dmpdir = self._base["DMPDIR"] - do_gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_cdumps else False - + def atmensanlinit(self): deps = [] - dep_dict = {'type': 'metatask', 'name': 'gdaspost', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009.nc' - dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} + dep_dict = {'type': 'task', 'name': f'{self.cdump.replace("enkf","")}prep'} deps.append(rocoto.add_dependency(dep_dict)) - data = f'{dmpdir}/{self.cdump}{dump_suffix}.@Y@m@d/@H/atmos/{self.cdump}.t@Hz.updated.status.tm00.bufr_d' - dep_dict = {'type': 'data', 'data': data} + dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - cycledef = self.cdump - if self.cdump in ['gfs'] and do_gfs_enkf and gfs_cyc != 4: - cycledef = 'gdas' - - resources = self.get_resource('atmensanalprep') - task = create_wf_task('atmensanalprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef = "gdas" + resources = self.get_resource('atmensanlinit') + task = create_wf_task('atmensanlinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, cycledef=cycledef) return task - def atmensanalrun(self): + def atmensanlrun(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanalprep'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlinit'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('atmensanalrun') - task = create_wf_task('atmensanalrun', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('atmensanlrun') + task = create_wf_task('atmensanlrun', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task - def atmensanalpost(self): + def atmensanlfinal(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanalrun'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlrun'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - resources = self.get_resource('atmensanalpost') - task = create_wf_task('atmensanalpost', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('atmensanlfinal') + task = create_wf_task('atmensanlfinal', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -1184,7 +1171,7 @@ def _get_ecengroups(): dep_dict = {'type': 'task', 'name': f'{self.cdump.replace("enkf","")}analcalc'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_jediens: - dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanalrun'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlrun'} else: dep_dict = {'type': 'task', 'name': f'{self.cdump}eupd'} deps.append(rocoto.add_dependency(dep_dict)) @@ -1213,7 +1200,7 @@ def esfc(self): dep_dict = {'type': 'task', 'name': f'{self.cdump.replace("enkf","")}analcalc'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_jediens: - dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanalrun'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlrun'} else: dep_dict = {'type': 'task', 'name': f'{self.cdump}eupd'} deps.append(rocoto.add_dependency(dep_dict)) From ccc418ac9ce06f3a3dcb9acd475652215672296c Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Mon, 6 Mar 2023 18:57:48 +0000 Subject: [PATCH 02/25] convert UFS-DA ATM ens increment to UFS model readable format (#1313) --- jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE | 2 +- ush/python/pygfs/task/atmens_analysis.py | 71 ++++++------------------ 2 files changed, 17 insertions(+), 56 deletions(-) diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE b/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE index bb7288ca369..5f3bf971150 100755 --- a/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE @@ -32,7 +32,7 @@ mkdir -p "${COMOUT}" # COMIN_GES and COMIN_GES_ENS are used in script export COMIN_GES="${ROTDIR}/${GDUMP}.${gPDY}/${gcyc}/atmos" -export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${gPDY}/${gcyc}/atmos" +export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${gPDY}/${gcyc}" ############################################################### # Run relevant script diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index a261bd3543a..79a4670750b 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -15,6 +15,7 @@ from pygw.template import Template, TemplateConstants from pygw.yaml_file import YAMLFile from pygw.logger import logit +from pygw.executable import Executable from pygfs.task.analysis import Analysis logger = getLogger(__name__.split('.')[-1]) @@ -120,12 +121,8 @@ def finalize(self: Analysis) -> None: This includes: - tarring up output diag files and place in ROTDIR - copying the generated YAML file from initialize to the ROTDIR - - copying the guess files to the ROTDIR - - applying the increments to the original RESTART files - - moving the increment files to the ROTDIR + - rewriting UFS-DA increments to UFS model readable format with delp and hydrostatic delz calculation - Please note that some of these steps are temporary and will be modified - once the model is able to read atmens tracer increments. """ # ---- tar up diags # path of output tar statfile @@ -153,70 +150,34 @@ def finalize(self: Analysis) -> None: } FileHandler(yaml_copy).sync() - # ---- NOTE below is 'temporary', eventually we will not be using FMS RESTART formatted files - # ---- all of the rest of this method will need to be changed but requires model and JEDI changes - # ---- copy RESTART fv_tracer files for future reference - # fms_bkg_file_template = os.path.join(self.task_config.comin_ges_atmens, 'RESTART', f'{self.task_config.cdate_fv3}.fv_tracer.res.tileX.nc') - # bkglist = [] - # for itile in range(1, self.task_config.ntiles + 1): - # bkg_path = fms_bkg_file_template.replace('tileX', f'tile{itile}') - # dest = os.path.join(self.task_config['COMOUT'], f'atmges.{os.path.basename(bkg_path)}') - # bkglist.append([bkg_path, dest]) - # FileHandler({'copy': bkglist}).sync() - - # ---- add increments to RESTART files - # logger.info('Adding increments to RESTART files') - # self._add_fms_cube_sphere_increments() - - # ---- move increments to ROTDIR - # logger.info('Moving increments to ROTDIR') - # fms_inc_file_template = os.path.join(self.task_config['DATA'], 'anl', f'atminc.{self.task_config.cdate_fv3}.fv_tracer.res.tileX.nc') - # inclist = [] - # for itile in range(1, self.task_config.ntiles + 1): - # inc_path = fms_inc_file_template.replace('tileX', f'tile{itile}') - # dest = os.path.join(self.task_config['COMOUT'], os.path.basename(inc_path)) - # inclist.append([inc_path, dest]) - # FileHandler({'copy': inclist}).sync() - - # ---- copy member increments to ROTDIR + # rewrite UFS-DA atmens increments to UFS model readable format with delp and hydrostatic delz calculation + gprefix = self.task_config['GPREFIX'] cdate_inc = self.task_config.cdate_fv3.replace('.', '_') - inclist = [] + incpy = os.path.join(self.task_config['HOMEgfs'], 'sorc/gdas.cd/ush/jediinc2fv3.py') + for imem in range(1, self.task_config['NMEM_ENKF'] + 1): memchar = f"mem{imem:03d}" - # make directory for member incrfement + # make output directory for member increment incdir = [ os.path.join(self.task_config['COMOUT'], memchar, 'atmos') ] FileHandler({'mkdir': incdir}).sync() - src = os.path.join(self.task_config['DATA'], 'anl', memchar, f'atminc.{cdate_inc}z.nc4') - dest = os.path.join(self.task_config['COMOUT'], memchar, 'atmos', f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.atminc.nc") - cdate_inc = self.task_config.cdate_fv3.replace('.', '_') - inclist.append([src, dest]) + # rewrite UFS-DA atmens increments + atmges_fv3 = os.path.join(self.task_config['COMIN_GES_ENS'], memchar, 'atmos', f"{self.task_config['CDUMP']}.t{self.task_config['gcyc']:02d}z.atmf006.nc") + atminc_jedi = os.path.join(self.task_config['DATA'], 'anl', memchar, f'atminc.{cdate_inc}z.nc4') + atminc_fv3 = os.path.join(self.task_config['COMOUT'], memchar, 'atmos', f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.atminc.nc") - inc_dict = { - 'copy': inclist, - } - FileHandler(inc_dict).sync() + cmd = Executable(incpy) + cmd.add_default_arg(atmges_fv3) + cmd.add_default_arg(atminc_jedi) + cmd.add_default_arg(atminc_fv3) + cmd(output='stdout', error='stderr') def clean(self): super().clean() - @logit(logger) - def _add_fms_cube_sphere_increments(self: Analysis) -> None: - """This method adds increments to RESTART files to get an analysis - NOTE this is only needed for now because the model cannot read atmens increments. - This method will be assumed to be deprecated before this is implemented operationally - """ - # only need the fv_tracer files - fms_inc_file_template = os.path.join(self.task_config['DATA'], 'anl', f'atminc.{self.task_config.cdate_fv3}.fv_tracer.res.tileX.nc') - fms_bkg_file_template = os.path.join(self.task_config.comin_ges_atm, 'RESTART', f'{self.task_config.cdate_fv3}.fv_tracer.res.tileX.nc') - # get list of increment vars - incvars_list_path = os.path.join(self.task_config['HOMEgfs'], 'parm', 'parm_gdas', 'atmensanl_inc_vars.yaml') - incvars = YAMLFile(path=incvars_list_path) - super().add_fv3_increments(fms_inc_file_template, fms_bkg_file_template, incvars) - @logit(logger) def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: """Compile a dictionary of model background files to copy From 10670fd73846d1127bd18b5a0edb8ec129b52996 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Mon, 6 Mar 2023 19:01:02 +0000 Subject: [PATCH 03/25] correct pynorms errors in atmens_analysis.py (#1313) --- ush/python/pygfs/task/atmens_analysis.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 79a4670750b..a34af46b9c6 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -165,9 +165,11 @@ def finalize(self: Analysis) -> None: FileHandler({'mkdir': incdir}).sync() # rewrite UFS-DA atmens increments - atmges_fv3 = os.path.join(self.task_config['COMIN_GES_ENS'], memchar, 'atmos', f"{self.task_config['CDUMP']}.t{self.task_config['gcyc']:02d}z.atmf006.nc") + atmges_fv3 = os.path.join(self.task_config['COMIN_GES_ENS'], memchar, 'atmos', + f"{self.task_config['CDUMP']}.t{self.task_config['gcyc']:02d}z.atmf006.nc") atminc_jedi = os.path.join(self.task_config['DATA'], 'anl', memchar, f'atminc.{cdate_inc}z.nc4') - atminc_fv3 = os.path.join(self.task_config['COMOUT'], memchar, 'atmos', f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.atminc.nc") + atminc_fv3 = os.path.join(self.task_config['COMOUT'], memchar, 'atmos', + f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.atminc.nc") cmd = Executable(incpy) cmd.add_default_arg(atmges_fv3) From 13acd1e84927a91985294f8635f96c43d17bb447 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 7 Mar 2023 21:45:12 +0000 Subject: [PATCH 04/25] update UFS-DA ATM ens ecen and esfc job dependencies (#1313) --- workflow/rocoto/workflow_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py index b2e22dd074b..d13c830d3d7 100644 --- a/workflow/rocoto/workflow_tasks.py +++ b/workflow/rocoto/workflow_tasks.py @@ -1171,7 +1171,7 @@ def _get_ecengroups(): dep_dict = {'type': 'task', 'name': f'{self.cdump.replace("enkf","")}analcalc'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_jediens: - dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlrun'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlfinal'} else: dep_dict = {'type': 'task', 'name': f'{self.cdump}eupd'} deps.append(rocoto.add_dependency(dep_dict)) @@ -1200,7 +1200,7 @@ def esfc(self): dep_dict = {'type': 'task', 'name': f'{self.cdump.replace("enkf","")}analcalc'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_jediens: - dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlrun'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlfinal'} else: dep_dict = {'type': 'task', 'name': f'{self.cdump}eupd'} deps.append(rocoto.add_dependency(dep_dict)) From 8adfaf7e76a88bf7f581653058520714f6b07e37 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 17 Mar 2023 14:35:04 +0000 Subject: [PATCH 05/25] clean up comin variables in UFS-DA ens jobs, remove DATA from ens diagnostic file tarball path (#1313) --- jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE | 2 +- jobs/JGLOBAL_ATMENS_ANALYSIS_RUN | 6 +----- ush/python/pygfs/task/atmens_analysis.py | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE index 8dcd0e9fb91..e9ca8083489 100755 --- a/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE @@ -25,7 +25,7 @@ export OPREFIX="${CDUMP_OBS}.t${cyc}z." export GPREFIX="${GDUMP}.t${gcyc}z." export APREFIX="${CDUMP}.t${cyc}z." -export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/atmos} +export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}} mkdir -p "${COMOUT}" diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN b/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN index a8dbf4aeb41..c196a61fc03 100755 --- a/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN @@ -26,14 +26,10 @@ export OPREFIX="${CDUMP}.t${cyc}z." export GPREFIX="${GDUMP}.t${gcyc}z." export APREFIX="${CDUMP}.t${cyc}z." -export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/atmos} +export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}} mkdir -p "${COMOUT}" -# COMIN_GES and COMIN_GES_ENS are used in script -export COMIN_GES="${ROTDIR}/${GDUMP}.${gPDY}/${gcyc}/atmos" -export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${gPDY}/${gcyc}/atmos" - ############################################################### # Run relevant script diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index a34af46b9c6..d0c0efa2ec1 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -129,7 +129,7 @@ def finalize(self: Analysis) -> None: atmensstat = os.path.join(self.task_config['COMOUT'], f"{self.task_config['APREFIX']}atmensstat") # get list of diag files to put in tarball - diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc4')) + diags = glob.glob('diags/diag*nc4') # gzip the files first for diagfile in diags: From 6e686c2bbd5be171f463a2da4172e3de104da4cb Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 17 Mar 2023 14:44:58 +0000 Subject: [PATCH 06/25] export gPDY in UFS-DA ens run and finalize jobs (#1313) --- jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE | 2 +- jobs/JGLOBAL_ATMENS_ANALYSIS_RUN | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE b/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE index 5f3bf971150..14e867e7488 100755 --- a/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE @@ -18,7 +18,7 @@ export CDUMP=${CDUMP:-${RUN:-"gfs"}} GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") export GDATE -gPDY=${GDATE:0:8} +export gPDY=${GDATE:0:8} export gcyc=${GDATE:8:2} export GDUMP=${GDUMP:-"gdas"} diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN b/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN index c196a61fc03..baac93cc42b 100755 --- a/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN @@ -18,7 +18,7 @@ export CDUMP=${CDUMP:-${RUN:-"gfs"}} GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") export GDATE -gPDY=${GDATE:0:8} +export gPDY=${GDATE:0:8} export gcyc=${GDATE:8:2} export GDUMP=${GDUMP:-"gdas"} From f1bedf5ad8291dc31e9d161117752468def9e7e4 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Mon, 27 Mar 2023 16:29:35 +0000 Subject: [PATCH 07/25] update UFS-DA ATM ens crtm and fix stage (#1313) --- ush/python/pygfs/task/atmens_analysis.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index d0c0efa2ec1..4292d8f54c4 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -75,10 +75,16 @@ def initialize(self: Analysis) -> None: super().initialize() # stage CRTM fix files - super().stage_fix('atm_crtm_coeff.yaml') + crtm_fix_list_path = os.path.join(self.task_config['HOMEgfs'], 'parm', 'parm_gdas', 'atm_crtm_coeff.yaml') + logger.debug(f"Staging CRTM fix files from {crtm_fix_list_path}") + crtm_fix_list = parse_yamltmpl(crtm_fix_list_path, self.task_config) + FileHandler(crtm_fix_list).sync() # stage fix files - super().stage_fix('atm_jedi_fix.yaml') + jedi_fix_list_path = os.path.join(self.task_config['HOMEgfs'], 'parm', 'parm_gdas', 'atm_jedi_fix.yaml') + logger.debug(f"Staging JEDI fix files from {jedi_fix_list_path}") + jedi_fix_list = parse_yamltmpl(jedi_fix_list_path, self.task_config) + FileHandler(jedi_fix_list).sync() # stage backgrounds FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() From 036b1f03f6648cafd44086cb2b6732664045bc0f Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Mon, 27 Mar 2023 19:16:30 +0000 Subject: [PATCH 08/25] import additional functions from pygw.yaml_file in atmens_analysis.py (#1313) --- ush/python/pygfs/task/atmens_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 4292d8f54c4..fbc1ab9acf4 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -13,7 +13,7 @@ from pygw.timetools import to_isotime, to_fv3time, to_timedelta from pygw.fsutils import rm_p from pygw.template import Template, TemplateConstants -from pygw.yaml_file import YAMLFile +from pygw.yaml_file import YAMLFile, parse_yamltmpl, parse_j2yaml, save_as_yaml from pygw.logger import logit from pygw.executable import Executable from pygfs.task.analysis import Analysis From 07e4e29669ac9b00298e9fbbaa81f779002dea53 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 30 Mar 2023 23:17:58 +0000 Subject: [PATCH 09/25] sync UFS-DA ATM ens jobs and atmens_analysis task with g-w python (#1313) --- jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE | 22 +---- jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE | 22 +---- jobs/JGLOBAL_ATMENS_ANALYSIS_RUN | 18 +--- ush/python/pygfs/task/atmens_analysis.py | 115 +++++++++++++---------- 4 files changed, 78 insertions(+), 99 deletions(-) diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE b/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE index 14e867e7488..e6989ef1a6e 100755 --- a/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE @@ -8,31 +8,19 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlfinal" -c "base atmensanl at ############################################## # Set variables used in the script ############################################## -export CDATE=${CDATE:-${PDY}${cyc}} -export CDUMP=${CDUMP:-${RUN:-"gfs"}} - +GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +GDUMP="gdas" ############################################## # Begin JOB SPECIFIC work ############################################## -GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") -export GDATE -export gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} -export GDUMP=${GDUMP:-"gdas"} - -export OPREFIX="${CDUMP}.t${cyc}z." -export GPREFIX="${GDUMP}.t${gcyc}z." -export APREFIX="${CDUMP}.t${cyc}z." - -export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}} - +export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}} mkdir -p "${COMOUT}" # COMIN_GES and COMIN_GES_ENS are used in script -export COMIN_GES="${ROTDIR}/${GDUMP}.${gPDY}/${gcyc}/atmos" -export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${gPDY}/${gcyc}" +export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos" +export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}" ############################################################### # Run relevant script diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE index e9ca8083489..dca7d0ffc64 100755 --- a/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE @@ -7,31 +7,19 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlinit" -c "base atmensanl atm ############################################## # Set variables used in the script ############################################## -export CDATE=${CDATE:-${PDY}${cyc}} -export CDUMP=${CDUMP:-${RUN:-"gfs"}} +GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +GDUMP="gdas" ############################################## # Begin JOB SPECIFIC work ############################################## - -GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") -export GDATE -export gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} -export GDUMP=${GDUMP:-"gdas"} - -export OPREFIX="${CDUMP_OBS}.t${cyc}z." -export GPREFIX="${GDUMP}.t${gcyc}z." -export APREFIX="${CDUMP}.t${cyc}z." - -export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}} - +export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}} mkdir -p "${COMOUT}" # COMIN_GES and COMIN_GES_ENS are used in script -export COMIN_GES="${ROTDIR}/${GDUMP}.${gPDY}/${gcyc}/atmos" -export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${gPDY}/${gcyc}" +export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos" +export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}" ############################################################### # Run relevant script diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN b/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN index baac93cc42b..4b2f143efae 100755 --- a/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN @@ -8,26 +8,14 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlrun" -c "base atmensanl atme ############################################## # Set variables used in the script ############################################## -export CDATE=${CDATE:-${PDY}${cyc}} -export CDUMP=${CDUMP:-${RUN:-"gfs"}} - +GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +GDUMP="gdas" ############################################## # Begin JOB SPECIFIC work ############################################## -GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") -export GDATE -export gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} -export GDUMP=${GDUMP:-"gdas"} - -export OPREFIX="${CDUMP}.t${cyc}z." -export GPREFIX="${GDUMP}.t${gcyc}z." -export APREFIX="${CDUMP}.t${cyc}z." - -export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}} - +export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}} mkdir -p "${COMOUT}" ############################################################### diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index fbc1ab9acf4..2ee6bc8bb60 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -4,18 +4,17 @@ import glob import gzip import tarfile -from netCDF4 import Dataset from logging import getLogger from typing import Dict, List, Any from pygw.attrdict import AttrDict from pygw.file_utils import FileHandler -from pygw.timetools import to_isotime, to_fv3time, to_timedelta -from pygw.fsutils import rm_p -from pygw.template import Template, TemplateConstants +from pygw.timetools import add_to_datetime, to_fv3time, to_timedelta, to_YMDH +from pygw.fsutils import rm_p, chdir from pygw.yaml_file import YAMLFile, parse_yamltmpl, parse_j2yaml, save_as_yaml from pygw.logger import logit from pygw.executable import Executable +from pygw.exceptions import WorkflowException from pygfs.task.analysis import Analysis logger = getLogger(__name__.split('.')[-1]) @@ -31,7 +30,8 @@ def __init__(self, config): _res = int(self.config['CASE_ANL'][1:]) _res_enkf = int(self.config['CASE_ENKF'][1:]) - _window_begin = self.runtime_config.current_cycle - to_timedelta(f"{self.config['assim_freq']}H") / 2 + _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) + _fv3jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.atmens.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -43,13 +43,14 @@ def __init__(self, config): 'npx_anl': _res_enkf + 1, 'npy_anl': _res_enkf + 1, 'npz_anl': self.config['LEVS'] - 1, - 'ATM_WINDOW_BEGIN': to_isotime(_window_begin), + 'ATM_WINDOW_BEGIN': _window_begin, 'ATM_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H", - 'BKG_ISOTIME': to_isotime(self.runtime_config.current_cycle), - 'BKG_YYYYmmddHHMMSS': to_fv3time(self.runtime_config.current_cycle), - 'cdate_fv3': to_fv3time(self.runtime_config.current_cycle), 'comin_ges_atm': self.config.COMIN_GES, 'comin_ges_atmens': self.config.COMIN_GES_ENS, + 'OPREFIX': f"{self.config.EUPD_CYC}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", + 'fv3jedi_yaml': _fv3jedi_yaml, } ) @@ -70,8 +71,6 @@ def initialize(self: Analysis) -> None: - linking the JEDI executable (TODO make it copyable, requires JEDI fix) - creating output directories """ - - # stage observations and bias corrections super().initialize() # stage CRTM fix files @@ -89,29 +88,29 @@ def initialize(self: Analysis) -> None: # stage backgrounds FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() - # generate variational YAML file - yaml_out = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.atmens.yaml") - varda_yaml = YAMLFile(path=self.task_config['ATMENSYAML']) - varda_yaml = Template.substitute_structure(varda_yaml, TemplateConstants.DOUBLE_CURLY_BRACES, self.task_config.get) - varda_yaml = Template.substitute_structure(varda_yaml, TemplateConstants.DOLLAR_PARENTHESES, self.task_config.get) - varda_yaml.save(yaml_out) - logger.info(f"Wrote YAML to {yaml_out}") + # generate ensemble da YAML file + logger.debug(f"Generate ensemble da YAML file: {self.task_config.fv3jedi_yaml}") + ensda_yaml = parse_j2yaml(self.task_config['ATMENSYAML'], self.task_config) + save_as_yaml(ensda_yaml, self.task_config.fv3jedi_yaml) + logger.info(f"Wrote ensemble da YAML to: {self.task_config.fv3jedi_yaml}") - # link var executable + # link executable to DATA/ directory exe_src = self.task_config['JEDIENSEXE'] + logger.debug(f"Link executable {exe_src} to DATA/") # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. exe_dest = os.path.join(self.task_config['DATA'], os.path.basename(exe_src)) if os.path.exists(exe_dest): rm_p(exe_dest) os.symlink(exe_src, exe_dest) # need output dir for diags and anl + logger.debug("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ os.path.join(self.task_config['DATA'], 'anl'), os.path.join(self.task_config['DATA'], 'diags'), ] FileHandler({'mkdir': newdirs}).sync() - # Make directories for member analsis files + # Make directories for member analysis files for imem in range(1, self.task_config['NMEM_ENKF'] + 1): memchar = f"mem{imem:03d}" anldir = [ @@ -119,23 +118,45 @@ def initialize(self: Analysis) -> None: ] FileHandler({'mkdir': anldir}).sync() + @logit(logger) + def execute(self: Analysis) -> None: + + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_ATMENSANL) + exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_letkf.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg(self.task_config.fv3jedi_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + @logit(logger) def finalize(self: Analysis) -> None: """Finalize a global atmens analysis This method will finalize a global atmens analysis using JEDI. This includes: - - tarring up output diag files and place in ROTDIR - - copying the generated YAML file from initialize to the ROTDIR - - rewriting UFS-DA increments to UFS model readable format with delp and hydrostatic delz calculation + - tar output diag files and place in ROTDIR + - copy the generated YAML file from initialize to the ROTDIR + - rewrite UFS-DA increments to UFS model readable format with delp and hydrostatic delz calculation + Please note that some of these steps are temporary and will be modified + once the model is able to read atm increments. """ # ---- tar up diags # path of output tar statfile atmensstat = os.path.join(self.task_config['COMOUT'], f"{self.task_config['APREFIX']}atmensstat") # get list of diag files to put in tarball - diags = glob.glob('diags/diag*nc4') + diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc4')) # gzip the files first for diagfile in diags: @@ -145,7 +166,8 @@ def finalize(self: Analysis) -> None: # open tar file for writing with tarfile.open(atmensstat, "w") as archive: for diagfile in diags: - archive.add(f"{diagfile}.gz") + diaggzip = f"{diagfile}.gz" + archive.add(diaggzip, arcname=os.path.basename(diaggzip)) # copy full YAML from executable to ROTDIR src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.atmens.yaml") @@ -158,7 +180,8 @@ def finalize(self: Analysis) -> None: # rewrite UFS-DA atmens increments to UFS model readable format with delp and hydrostatic delz calculation gprefix = self.task_config['GPREFIX'] - cdate_inc = self.task_config.cdate_fv3.replace('.', '_') + cdate = to_fv3time(self.task_config.current_cycle) + cdate_inc = cdate.replace('.', '_') incpy = os.path.join(self.task_config['HOMEgfs'], 'sorc/gdas.cd/ush/jediinc2fv3.py') for imem in range(1, self.task_config['NMEM_ENKF'] + 1): @@ -172,7 +195,7 @@ def finalize(self: Analysis) -> None: # rewrite UFS-DA atmens increments atmges_fv3 = os.path.join(self.task_config['COMIN_GES_ENS'], memchar, 'atmos', - f"{self.task_config['CDUMP']}.t{self.task_config['gcyc']:02d}z.atmf006.nc") + f"{self.task_config['CDUMP']}.t{self.runtime_config.previous_cycle.hour:02d}z.atmf006.nc") atminc_jedi = os.path.join(self.task_config['DATA'], 'anl', memchar, f'atminc.{cdate_inc}z.nc4') atminc_fv3 = os.path.join(self.task_config['COMOUT'], memchar, 'atmos', f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.atminc.nc") @@ -203,7 +226,6 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: bkg_dict: Dict a dictionary containing the list of model background files to copy for FileHandler """ - super().get_bkg_dict(task_config) # NOTE for now this is FV3 RESTART files and just assumed to be fh006 bkgdir = [ @@ -224,32 +246,25 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: # get FV3 RESTART files, this will be a lot simpler when using history files rst_dir = os.path.join(task_config.comin_ges_atmens, memchar, 'atmos/RESTART') + run_dir = os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART') - basename = f'{task_config.cdate_fv3}.coupler.res' + # atmens DA needs coupler + basename = f'{to_fv3time(task_config.current_cycle)}.coupler.res' bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) - basename = f'{task_config.cdate_fv3}.fv_core.res.nc' - bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) - basename_cadat = f'{task_config.cdate_fv3}.ca_data.tileX.nc' - basename_core = f'{task_config.cdate_fv3}.fv_core.res.tileX.nc' - basename_srfwnd = f'{task_config.cdate_fv3}.fv_srf_wnd.res.tileX.nc' - basename_tracer = f'{task_config.cdate_fv3}.fv_tracer.res.tileX.nc' - basename_phydat = f'{task_config.cdate_fv3}.phy_data.tileX.nc' - basename_sfcdat = f'{task_config.cdate_fv3}.sfc_data.tileX.nc' - for itile in range(1, task_config.ntiles + 1): - basename = basename_cadat.replace('tileX', f'tile{itile}') - bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) - basename = basename_core.replace('tileX', f'tile{itile}') - bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) - basename = basename_srfwnd.replace('tileX', f'tile{itile}') - bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) - basename = basename_tracer.replace('tileX', f'tile{itile}') - bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) - basename = basename_phydat.replace('tileX', f'tile{itile}') - bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) - basename = basename_sfcdat.replace('tileX', f'tile{itile}') - bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) + # atmens DA needs core, srf_wnd, tracer, phy_data, sfc_data + for ftype in ['core', 'srf_wnd', 'tracer']: + template = f'{to_fv3time(self.task_config.current_cycle)}.fv_{ftype}.res.tile{{tilenum}}.nc' + for itile in range(1, task_config.ntiles + 1): + basename = template.format(tilenum=itile) + bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) + for ftype in ['phy_data', 'sfc_data']: + template = f'{to_fv3time(self.task_config.current_cycle)}.{ftype}.tile{{tilenum}}.nc' + for itile in range(1, task_config.ntiles + 1): + basename = template.format(tilenum=itile) + bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) bkg_dict = { 'copy': bkglist, } + return bkg_dict From e31aa75832af4de8836af35fcfcb32da290a0cbe Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 30 Mar 2023 23:44:40 +0000 Subject: [PATCH 10/25] address linter errors in JGLOBAL_ATMENS_ANALYSIS_RUN (#1313) --- jobs/JGLOBAL_ATMENS_ANALYSIS_RUN | 2 -- 1 file changed, 2 deletions(-) diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN b/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN index 4b2f143efae..eed256a7f3d 100755 --- a/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN @@ -8,8 +8,6 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlrun" -c "base atmensanl atme ############################################## # Set variables used in the script ############################################## -GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -GDUMP="gdas" ############################################## # Begin JOB SPECIFIC work From 9b4fbbfdc2757d3c9ab0a0f75c4891588c95f3b3 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 5 Apr 2023 18:09:18 +0000 Subject: [PATCH 11/25] remove RESTART from ensemble member background run directory path (#1313) --- ush/python/pygfs/task/atmens_analysis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 2ee6bc8bb60..acd12ca1512 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -240,17 +240,17 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: # make run directory for member restart files bkgdir = [ - os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART') + os.path.join(task_config['DATA'], 'bkg', memchar) ] FileHandler({'mkdir': bkgdir}).sync() # get FV3 RESTART files, this will be a lot simpler when using history files rst_dir = os.path.join(task_config.comin_ges_atmens, memchar, 'atmos/RESTART') - run_dir = os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART') + run_dir = os.path.join(task_config['DATA'], 'bkg', memchar) # atmens DA needs coupler basename = f'{to_fv3time(task_config.current_cycle)}.coupler.res' - bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, 'RESTART', basename)]) + bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, basename)]) # atmens DA needs core, srf_wnd, tracer, phy_data, sfc_data for ftype in ['core', 'srf_wnd', 'tracer']: template = f'{to_fv3time(self.task_config.current_cycle)}.fv_{ftype}.res.tile{{tilenum}}.nc' From 00baac52eda1d33dd23a221faa2bb19a25ef8e33 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Mon, 17 Apr 2023 15:26:59 +0000 Subject: [PATCH 12/25] update atmens python scripting (#1313) --- jobs/JGLOBAL_ATMENS_ANALYSIS_RUN | 2 +- jobs/rocoto/atmensanlrun.sh | 6 + scripts/exglobal_atmens_analysis_run.py | 23 ++++ scripts/exglobal_atmens_analysis_run.sh | 16 --- ush/python/pygfs/task/atmens_analysis.py | 146 +++++++++++++---------- 5 files changed, 116 insertions(+), 77 deletions(-) create mode 100755 scripts/exglobal_atmens_analysis_run.py delete mode 100755 scripts/exglobal_atmens_analysis_run.sh diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN b/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN index eed256a7f3d..5a267f197a8 100755 --- a/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN @@ -19,7 +19,7 @@ mkdir -p "${COMOUT}" ############################################################### # Run relevant script -EXSCRIPT=${GDASATMENSRUNSH:-${HOMEgfs}/scripts/exglobal_atmens_analysis_run.sh} +EXSCRIPT=${GDASATMENSRUNSH:-${HOMEgfs}/scripts/exglobal_atmens_analysis_run.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/atmensanlrun.sh b/jobs/rocoto/atmensanlrun.sh index 8bfbc7f6629..91efdb37686 100755 --- a/jobs/rocoto/atmensanlrun.sh +++ b/jobs/rocoto/atmensanlrun.sh @@ -11,6 +11,12 @@ status=$? export job="atmensanlrun" export jobid="${job}.$$" +############################################################### +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN" diff --git a/scripts/exglobal_atmens_analysis_run.py b/scripts/exglobal_atmens_analysis_run.py new file mode 100755 index 00000000000..afc4139f4ea --- /dev/null +++ b/scripts/exglobal_atmens_analysis_run.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# exgdas_global_atmens_analysis_run.py +# This script creates an AtmEnsAnalysis object +# and runs the execute method +# which executes the global atm ensemble analysis +import os + +from pygw.logger import Logger +from pygw.configuration import cast_strdict_as_dtypedict +from pygfs.task.atmens_analysis import AtmEnsAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the atmens analysis task + AtmEnsAnl = AtmEnsAnalysis(config) + AtmEnsAnl.execute() diff --git a/scripts/exglobal_atmens_analysis_run.sh b/scripts/exglobal_atmens_analysis_run.sh deleted file mode 100755 index 19ce9017102..00000000000 --- a/scripts/exglobal_atmens_analysis_run.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -################################################################################ -# exgdas_global_atmens_analysis_run.sh -# -# This script runs a global atmens variational analysis with FV3-JEDI. -# It assumes the runtime directory has already been staged with the appropriate -# input files and YAML configuration (by the initialize script) before execution. -# -################################################################################ -# run executable -set -x -export pgm=${JEDIVAREXE} -. prep_step -${APRUN_ATMENSANL} "${DATA}/fv3jedi_letkf.x" "${DATA}/${CDUMP}.t${cyc}z.atmens.yaml" 1>&1 2>&2 -export err=$?; err_chk -exit "${err}" diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index acd12ca1512..72420023ffc 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -11,7 +11,7 @@ from pygw.file_utils import FileHandler from pygw.timetools import add_to_datetime, to_fv3time, to_timedelta, to_YMDH from pygw.fsutils import rm_p, chdir -from pygw.yaml_file import YAMLFile, parse_yamltmpl, parse_j2yaml, save_as_yaml +from pygw.yaml_file import parse_yamltmpl, parse_j2yaml, save_as_yaml from pygw.logger import logit from pygw.executable import Executable from pygw.exceptions import WorkflowException @@ -28,10 +28,10 @@ class AtmEnsAnalysis(Analysis): def __init__(self, config): super().__init__(config) - _res = int(self.config['CASE_ANL'][1:]) - _res_enkf = int(self.config['CASE_ENKF'][1:]) - _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) - _fv3jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.atmens.yaml") + _res = int(self.config.CASE_ENKF[1:]) + _res_anl = int(self.config.CASE_ANL[1:]) + _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config.assim_freq}H") / 2) + _fv3jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atmens.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -40,11 +40,11 @@ def __init__(self, config): 'npy_ges': _res + 1, 'npz_ges': self.config.LEVS - 1, 'npz': self.config.LEVS - 1, - 'npx_anl': _res_enkf + 1, - 'npy_anl': _res_enkf + 1, - 'npz_anl': self.config['LEVS'] - 1, + 'npx_anl': _res_anl + 1, + 'npy_anl': _res_anl + 1, + 'npz_anl': self.config.LEVS - 1, 'ATM_WINDOW_BEGIN': _window_begin, - 'ATM_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H", + 'ATM_WINDOW_LENGTH': f"PT{self.config.assim_freq}H", 'comin_ges_atm': self.config.COMIN_GES, 'comin_ges_atmens': self.config.COMIN_GES_ENS, 'OPREFIX': f"{self.config.EUPD_CYC}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN @@ -65,7 +65,6 @@ def initialize(self: Analysis) -> None: This includes: - staging CRTM fix files - staging FV3-JEDI fix files - - staging B error files - staging model backgrounds - generating a YAML file for the JEDI executable - linking the JEDI executable (TODO make it copyable, requires JEDI fix) @@ -74,30 +73,30 @@ def initialize(self: Analysis) -> None: super().initialize() # stage CRTM fix files - crtm_fix_list_path = os.path.join(self.task_config['HOMEgfs'], 'parm', 'parm_gdas', 'atm_crtm_coeff.yaml') + crtm_fix_list_path = os.path.join(self.task_config.HOMEgfs, 'parm', 'parm_gdas', 'atm_crtm_coeff.yaml') logger.debug(f"Staging CRTM fix files from {crtm_fix_list_path}") crtm_fix_list = parse_yamltmpl(crtm_fix_list_path, self.task_config) FileHandler(crtm_fix_list).sync() # stage fix files - jedi_fix_list_path = os.path.join(self.task_config['HOMEgfs'], 'parm', 'parm_gdas', 'atm_jedi_fix.yaml') + jedi_fix_list_path = os.path.join(self.task_config.HOMEgfs, 'parm', 'parm_gdas', 'atm_jedi_fix.yaml') logger.debug(f"Staging JEDI fix files from {jedi_fix_list_path}") jedi_fix_list = parse_yamltmpl(jedi_fix_list_path, self.task_config) FileHandler(jedi_fix_list).sync() # stage backgrounds - FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() + FileHandler(self.get_bkg_dict(AttrDict(self.task_config))).sync() # generate ensemble da YAML file logger.debug(f"Generate ensemble da YAML file: {self.task_config.fv3jedi_yaml}") - ensda_yaml = parse_j2yaml(self.task_config['ATMENSYAML'], self.task_config) + ensda_yaml = parse_j2yaml(self.task_config.ATMENSYAML, self.task_config) save_as_yaml(ensda_yaml, self.task_config.fv3jedi_yaml) logger.info(f"Wrote ensemble da YAML to: {self.task_config.fv3jedi_yaml}") # link executable to DATA/ directory - exe_src = self.task_config['JEDIENSEXE'] + exe_src = self.task_config.JEDIENSEXE logger.debug(f"Link executable {exe_src} to DATA/") # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. - exe_dest = os.path.join(self.task_config['DATA'], os.path.basename(exe_src)) + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) if os.path.exists(exe_dest): rm_p(exe_dest) os.symlink(exe_src, exe_dest) @@ -105,16 +104,16 @@ def initialize(self: Analysis) -> None: # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ - os.path.join(self.task_config['DATA'], 'anl'), - os.path.join(self.task_config['DATA'], 'diags'), + os.path.join(self.task_config.DATA, 'anl'), + os.path.join(self.task_config.DATA, 'diags'), ] FileHandler({'mkdir': newdirs}).sync() # Make directories for member analysis files - for imem in range(1, self.task_config['NMEM_ENKF'] + 1): + for imem in range(1, self.task_config.NMEM_ENKF + 1): memchar = f"mem{imem:03d}" anldir = [ - os.path.join(self.task_config['DATA'], 'anl', memchar) + os.path.join(self.task_config.DATA, 'anl', memchar) ] FileHandler({'mkdir': anldir}).sync() @@ -146,65 +145,45 @@ def finalize(self: Analysis) -> None: This includes: - tar output diag files and place in ROTDIR - copy the generated YAML file from initialize to the ROTDIR - - rewrite UFS-DA increments to UFS model readable format with delp and hydrostatic delz calculation + - write UFS model readable atm incrment file - Please note that some of these steps are temporary and will be modified - once the model is able to read atm increments. """ # ---- tar up diags # path of output tar statfile - atmensstat = os.path.join(self.task_config['COMOUT'], f"{self.task_config['APREFIX']}atmensstat") + atmensstat = os.path.join(self.task_config.COMOUT, f"{self.task_config.APREFIX}atmensstat") # get list of diag files to put in tarball - diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc4')) + diags = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc4')) + + logger.info(f"Compressing {len(diags)} diag files to {atmensstat}.gz") # gzip the files first + logger.debug(f"Gzipping {len(diags)} diag files") for diagfile in diags: with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: f_out.writelines(f_in) # open tar file for writing + logger.debug(f"Creating tar file {atmensstat} with {len(diags)} gzipped diag files") with tarfile.open(atmensstat, "w") as archive: for diagfile in diags: diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) # copy full YAML from executable to ROTDIR - src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.atmens.yaml") - dest = os.path.join(self.task_config['COMOUT'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.atmens.yaml") + logger.info(f"Copying {self.task_config.fv3jedi_yaml} to {self.task_config.COMOUT}") + src = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmens.yaml") + dest = os.path.join(self.task_config.COMOUT, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmens.yaml") + logger.debug(f"Copying {src} to {dest}") yaml_copy = { - 'mkdir': [self.task_config['COMOUT']], + 'mkdir': [self.task_config.COMOUT], 'copy': [[src, dest]] } FileHandler(yaml_copy).sync() - # rewrite UFS-DA atmens increments to UFS model readable format with delp and hydrostatic delz calculation - gprefix = self.task_config['GPREFIX'] - cdate = to_fv3time(self.task_config.current_cycle) - cdate_inc = cdate.replace('.', '_') - incpy = os.path.join(self.task_config['HOMEgfs'], 'sorc/gdas.cd/ush/jediinc2fv3.py') - - for imem in range(1, self.task_config['NMEM_ENKF'] + 1): - memchar = f"mem{imem:03d}" - - # make output directory for member increment - incdir = [ - os.path.join(self.task_config['COMOUT'], memchar, 'atmos') - ] - FileHandler({'mkdir': incdir}).sync() - - # rewrite UFS-DA atmens increments - atmges_fv3 = os.path.join(self.task_config['COMIN_GES_ENS'], memchar, 'atmos', - f"{self.task_config['CDUMP']}.t{self.runtime_config.previous_cycle.hour:02d}z.atmf006.nc") - atminc_jedi = os.path.join(self.task_config['DATA'], 'anl', memchar, f'atminc.{cdate_inc}z.nc4') - atminc_fv3 = os.path.join(self.task_config['COMOUT'], memchar, 'atmos', - f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.atminc.nc") - - cmd = Executable(incpy) - cmd.add_default_arg(atmges_fv3) - cmd.add_default_arg(atminc_jedi) - cmd.add_default_arg(atminc_fv3) - cmd(output='stdout', error='stderr') + # Create UFS model readable atm increment file from UFS-DA atm increment + logger.info("Create UFS model readable atm increment file from UFS-DA atm increment") + self.jedi2fv3inc() def clean(self): super().clean() @@ -229,34 +208,35 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: # NOTE for now this is FV3 RESTART files and just assumed to be fh006 bkgdir = [ - os.path.join(task_config['DATA'], 'bkg'), + os.path.join(task_config.DATA, 'bkg'), ] FileHandler({'mkdir': bkgdir}).sync() # loop over ensemble members bkglist = [] - for imem in range(1, task_config['NMEM_ENKF'] + 1): + for imem in range(1, task_config.NMEM_ENKF + 1): memchar = f"mem{imem:03d}" # make run directory for member restart files bkgdir = [ - os.path.join(task_config['DATA'], 'bkg', memchar) + os.path.join(task_config.DATA, 'bkg', memchar) ] FileHandler({'mkdir': bkgdir}).sync() # get FV3 RESTART files, this will be a lot simpler when using history files rst_dir = os.path.join(task_config.comin_ges_atmens, memchar, 'atmos/RESTART') - run_dir = os.path.join(task_config['DATA'], 'bkg', memchar) + run_dir = os.path.join(task_config.DATA, 'bkg', memchar) # atmens DA needs coupler basename = f'{to_fv3time(task_config.current_cycle)}.coupler.res' - bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config['DATA'], 'bkg', memchar, basename)]) + bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config.DATA, 'bkg', memchar, basename)]) # atmens DA needs core, srf_wnd, tracer, phy_data, sfc_data for ftype in ['core', 'srf_wnd', 'tracer']: template = f'{to_fv3time(self.task_config.current_cycle)}.fv_{ftype}.res.tile{{tilenum}}.nc' for itile in range(1, task_config.ntiles + 1): basename = template.format(tilenum=itile) bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) + for ftype in ['phy_data', 'sfc_data']: template = f'{to_fv3time(self.task_config.current_cycle)}.{ftype}.tile{{tilenum}}.nc' for itile in range(1, task_config.ntiles + 1): @@ -268,3 +248,49 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: } return bkg_dict + + @logit(logger) + def jedi2fv3inc(self: Analysis) -> None: + """Generate UFS model readable analysis increment + + This method writes a UFS DA atm increment in UFS model readable format. + This includes: + - write UFS-DA atm increments using variable names expected by UFS model + - compute and write delp increment + - compute and write hydrostatic delz increment + + Please note that some of these steps are temporary and will be modified + once the modle is able to directly read atm increments. + + """ + # Select the atm guess file based on the analysis and background resolutions + # Fields from the atm guess are used to compute the delp and delz increments + cdate = to_fv3time(self.task_config.current_cycle) + cdate_inc = cdate.replace('.', '_') + + # Reference the python script which does the actual work + incpy = os.path.join(self.task_config.HOMEgfs, 'ush/jediinc2fv3.py') + + for imem in range(1, self.task_config.NMEM_ENKF + 1): + memchar = f"mem{imem:03d}" + + # make output directory for member increment + incdir = [ + os.path.join(self.task_config.COMOUT, memchar, 'atmos') + ] + FileHandler({'mkdir': incdir}).sync() + + # rewrite UFS-DA atmens increments + atmges_fv3 = os.path.join(self.task_config.COMIN_GES_ENS, memchar, 'atmos', + f"{self.task_config.CDUMP}.t{self.runtime_config.previous_cycle.hour:02d}z.atmf006.nc") + atminc_jedi = os.path.join(self.task_config.DATA, 'anl', memchar, f'atminc.{cdate_inc}z.nc4') + atminc_fv3 = os.path.join(self.task_config.COMOUT, memchar, 'atmos', + f"{self.task_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atminc.nc") + + # Execute incpy to create the UFS model atm increment file + cmd = Executable(incpy) + cmd.add_default_arg(atmges_fv3) + cmd.add_default_arg(atminc_jedi) + cmd.add_default_arg(atminc_fv3) + logger.debug(f"Executing {cmd}") + cmd(output='stdout', error='stderr') From 0335389ed733aa08d5f268fcc2df487bc6ed78f9 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 19 Apr 2023 10:25:56 +0000 Subject: [PATCH 13/25] update sha-bang in atmens config files, correct atmens errors in workflow_task (#1313) --- parm/config/config.atmensanl | 2 +- parm/config/config.atmensanlfinal | 2 +- parm/config/config.atmensanlinit | 2 +- parm/config/config.atmensanlrun | 2 +- workflow/rocoto/workflow_tasks.py | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/parm/config/config.atmensanl b/parm/config/config.atmensanl index 59454db4b08..ec03673027f 100755 --- a/parm/config/config.atmensanl +++ b/parm/config/config.atmensanl @@ -1,4 +1,4 @@ -#!/bin/bash -x +#! /usr/bin/env bash ########## config.atmensanl ########## # configuration common to all atm ens analysis tasks diff --git a/parm/config/config.atmensanlfinal b/parm/config/config.atmensanlfinal index db3b8b69c97..5d8ec458c38 100755 --- a/parm/config/config.atmensanlfinal +++ b/parm/config/config.atmensanlfinal @@ -1,4 +1,4 @@ -#!/bin/bash -x +#! /usr/bin/env bash ########## config.atmensanlfinal ########## # Post Atm Ens Analysis specific diff --git a/parm/config/config.atmensanlinit b/parm/config/config.atmensanlinit index a9cd4c968df..34429023bbb 100755 --- a/parm/config/config.atmensanlinit +++ b/parm/config/config.atmensanlinit @@ -1,4 +1,4 @@ -#!/bin/bash -x +#! /usr/bin/env bash ########## config.atmensanlinit ########## # Pre Atm Ens Analysis specific diff --git a/parm/config/config.atmensanlrun b/parm/config/config.atmensanlrun index aec8e04ec03..01f211a17a0 100755 --- a/parm/config/config.atmensanlrun +++ b/parm/config/config.atmensanlrun @@ -1,4 +1,4 @@ -#!/bin/bash -x +#! /usr/bin/env bash ########## config.atmensanlrun ########## # Atm Ens Analysis specific diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py index 771201abc09..27ffa8e5531 100644 --- a/workflow/rocoto/workflow_tasks.py +++ b/workflow/rocoto/workflow_tasks.py @@ -16,7 +16,7 @@ class Tasks: 'ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalpost', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', - 'atmensanlprep', 'atmensanlrun', 'atmensanlpost', + 'atmensanlinit', 'atmensanlrun', 'atmensanlfinal', 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', 'fcst', 'post', 'ocnpost', 'vrfy', 'metp', 'postsnd', 'awips', 'gempak', @@ -1178,7 +1178,7 @@ def _get_ecengroups(): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump.replace("enkf","")}analcalc'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_jediens: + if self.app_config.do_jediatmens: dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlfinal'} else: dep_dict = {'type': 'task', 'name': f'{self.cdump}eupd'} @@ -1207,7 +1207,7 @@ def esfc(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump.replace("enkf","")}analcalc'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_jediens: + if self.app_config.do_jediatmens: dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlfinal'} else: dep_dict = {'type': 'task', 'name': f'{self.cdump}eupd'} From c48dd6692812b667cb7d9a4213dd4b448778b4f5 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 19 Apr 2023 15:27:20 +0000 Subject: [PATCH 14/25] clean up comments in jobs and scripts (#1313) --- jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE | 2 +- scripts/exglobal_atmens_analysis_finalize.py | 4 +-- .../exglobal_atmens_analysis_initialize.py | 2 +- scripts/exglobal_atmens_analysis_run.py | 2 +- ush/python/pygfs/task/atmens_analysis.py | 36 +++++++++++++++++++ 5 files changed, 41 insertions(+), 5 deletions(-) diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE b/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE index e6989ef1a6e..d40d79cf78e 100755 --- a/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE @@ -44,7 +44,7 @@ fi ########################################## # Remove the Temporary working directory ########################################## -cd "${DATAROOT}" || exit 1 +cd "${DATAROOT}" || ( echo "FATAL ERROR: ${DATAROOT} does not exist, ABORT!"; exit 1 ) [[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" exit 0 diff --git a/scripts/exglobal_atmens_analysis_finalize.py b/scripts/exglobal_atmens_analysis_finalize.py index eb41b180711..5271c5c486e 100755 --- a/scripts/exglobal_atmens_analysis_finalize.py +++ b/scripts/exglobal_atmens_analysis_finalize.py @@ -3,10 +3,10 @@ # This script creates an AtmEnsAnalysis class # and runs the finalize method # which perform post-processing and clean up activities -# for a global atmens variational analysis +# for a global atm local ensemble analysis import os -from pygw.logger import Logger, logit +from pygw.logger import Logger from pygw.configuration import cast_strdict_as_dtypedict from pygfs.task.atmens_analysis import AtmEnsAnalysis diff --git a/scripts/exglobal_atmens_analysis_initialize.py b/scripts/exglobal_atmens_analysis_initialize.py index f2776190c03..97326ddf3de 100755 --- a/scripts/exglobal_atmens_analysis_initialize.py +++ b/scripts/exglobal_atmens_analysis_initialize.py @@ -4,7 +4,7 @@ # and runs the initialize method # which create and stage the runtime directory # and create the YAML configuration -# for a global atmens variational analysis +# for a global atm local ensemble analysis import os from pygw.logger import Logger diff --git a/scripts/exglobal_atmens_analysis_run.py b/scripts/exglobal_atmens_analysis_run.py index afc4139f4ea..2de95e850da 100755 --- a/scripts/exglobal_atmens_analysis_run.py +++ b/scripts/exglobal_atmens_analysis_run.py @@ -2,7 +2,7 @@ # exgdas_global_atmens_analysis_run.py # This script creates an AtmEnsAnalysis object # and runs the execute method -# which executes the global atm ensemble analysis +# which executes the global atm local ensemble analysis import os from pygw.logger import Logger diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 72420023ffc..ddf79fcf8ac 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -69,6 +69,14 @@ def initialize(self: Analysis) -> None: - generating a YAML file for the JEDI executable - linking the JEDI executable (TODO make it copyable, requires JEDI fix) - creating output directories + + Parameters + ---------- + Analysis: parent class for GDAS task + + Returns + ---------- + None """ super().initialize() @@ -119,7 +127,21 @@ def initialize(self: Analysis) -> None: @logit(logger) def execute(self: Analysis) -> None: + """Execute a global atmens analysis + This method will execute a global atmens analysis using JEDI. + This includes: + - changing to the run directory + - running the global atmens analysis executable + + Parameters + ---------- + Analysis: parent class for GDAS task + + Returns + ---------- + None + """ chdir(self.task_config.DATA) exec_cmd = Executable(self.task_config.APRUN_ATMENSANL) @@ -147,6 +169,13 @@ def finalize(self: Analysis) -> None: - copy the generated YAML file from initialize to the ROTDIR - write UFS model readable atm incrment file + Parameters + ---------- + Analysis: parent class for GDAS task + + Returns + ---------- + None """ # ---- tar up diags # path of output tar statfile @@ -262,6 +291,13 @@ def jedi2fv3inc(self: Analysis) -> None: Please note that some of these steps are temporary and will be modified once the modle is able to directly read atm increments. + Parameters + ---------- + Analysis: parent class for GDAS task + + Returns + ---------- + None """ # Select the atm guess file based on the analysis and background resolutions # Fields from the atm guess are used to compute the delp and delz increments From f902370ad57b553bf4d4fa5861250ae23a68650c Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 19 Apr 2023 17:12:21 +0000 Subject: [PATCH 15/25] collect bkg member directories in list for mkdir (#1313) --- ush/python/pygfs/task/atmens_analysis.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index ddf79fcf8ac..6a8a7697c8a 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -242,15 +242,13 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: FileHandler({'mkdir': bkgdir}).sync() # loop over ensemble members + dirlist = [] bkglist = [] for imem in range(1, task_config.NMEM_ENKF + 1): memchar = f"mem{imem:03d}" - # make run directory for member restart files - bkgdir = [ - os.path.join(task_config.DATA, 'bkg', memchar) - ] - FileHandler({'mkdir': bkgdir}).sync() + # accumulate directory list for member restart files + dirlist.append(os.path.join(task_config.DATA, 'bkg', memchar)) # get FV3 RESTART files, this will be a lot simpler when using history files rst_dir = os.path.join(task_config.comin_ges_atmens, memchar, 'atmos/RESTART') @@ -273,6 +271,7 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) bkg_dict = { + 'mkdir': dirlist, 'copy': bkglist, } From d5871934cb75c09d8d07884c98398767b838868c Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 19 Apr 2023 17:26:01 +0000 Subject: [PATCH 16/25] Remove Filehandler from get_bkg_dict (#1313) --- ush/python/pygfs/task/atmens_analysis.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 6a8a7697c8a..7a9ed058749 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -235,12 +235,6 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: a dictionary containing the list of model background files to copy for FileHandler """ # NOTE for now this is FV3 RESTART files and just assumed to be fh006 - - bkgdir = [ - os.path.join(task_config.DATA, 'bkg'), - ] - FileHandler({'mkdir': bkgdir}).sync() - # loop over ensemble members dirlist = [] bkglist = [] From aa162d0c05a3ad8a1bfe3420922735432b46afc7 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Date: Wed, 19 Apr 2023 13:47:04 -0400 Subject: [PATCH 17/25] Streamline script creating list of ensemble backgrounds (#1313) Co-authored-by: Rahul Mahajan --- ush/python/pygfs/task/atmens_analysis.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 7a9ed058749..bf77546c1d5 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -252,14 +252,8 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: basename = f'{to_fv3time(task_config.current_cycle)}.coupler.res' bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config.DATA, 'bkg', memchar, basename)]) # atmens DA needs core, srf_wnd, tracer, phy_data, sfc_data - for ftype in ['core', 'srf_wnd', 'tracer']: - template = f'{to_fv3time(self.task_config.current_cycle)}.fv_{ftype}.res.tile{{tilenum}}.nc' - for itile in range(1, task_config.ntiles + 1): - basename = template.format(tilenum=itile) - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - for ftype in ['phy_data', 'sfc_data']: - template = f'{to_fv3time(self.task_config.current_cycle)}.{ftype}.tile{{tilenum}}.nc' + template = f'{to_fv3time(self.task_config.current_cycle)}.{ftype}.tile{{tilenum}}.nc' + for ftype in ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data']: for itile in range(1, task_config.ntiles + 1): basename = template.format(tilenum=itile) bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) From 8746b1924a174afc3c3f55ab0a2178e05e0e0343 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 19 Apr 2023 17:49:51 +0000 Subject: [PATCH 18/25] correct pycodestyle errors (#1313) --- ush/python/pygfs/task/atmens_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index bf77546c1d5..a1a2b38b36d 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -253,7 +253,7 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config.DATA, 'bkg', memchar, basename)]) # atmens DA needs core, srf_wnd, tracer, phy_data, sfc_data template = f'{to_fv3time(self.task_config.current_cycle)}.{ftype}.tile{{tilenum}}.nc' - for ftype in ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data']: + for ftype in ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data']: for itile in range(1, task_config.ntiles + 1): basename = template.format(tilenum=itile) bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) From 6b66312cceb08736dc1d4386461fb1105a4ad624 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 19 Apr 2023 18:01:36 +0000 Subject: [PATCH 19/25] define ftype before use (#1313) --- ush/python/pygfs/task/atmens_analysis.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index a1a2b38b36d..86563658207 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -251,9 +251,10 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: # atmens DA needs coupler basename = f'{to_fv3time(task_config.current_cycle)}.coupler.res' bkglist.append([os.path.join(rst_dir, basename), os.path.join(task_config.DATA, 'bkg', memchar, basename)]) + # atmens DA needs core, srf_wnd, tracer, phy_data, sfc_data - template = f'{to_fv3time(self.task_config.current_cycle)}.{ftype}.tile{{tilenum}}.nc' for ftype in ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data']: + template = f'{to_fv3time(self.task_config.current_cycle)}.{ftype}.tile{{tilenum}}.nc' for itile in range(1, task_config.ntiles + 1): basename = template.format(tilenum=itile) bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) From 4b61129abbe17220c51ef028f66d945b1ad40758 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 19 Apr 2023 21:39:11 +0000 Subject: [PATCH 20/25] move JEDI executable link to analysis.py (#1313) --- parm/config/config.aeroanl | 2 +- parm/config/config.atmanl | 2 +- parm/config/config.atmensanl | 2 +- ush/python/pygfs/task/aero_analysis.py | 9 ------- ush/python/pygfs/task/analysis.py | 30 +++++++++++++++++++++++- ush/python/pygfs/task/atm_analysis.py | 9 ------- ush/python/pygfs/task/atmens_analysis.py | 9 ------- 7 files changed, 32 insertions(+), 31 deletions(-) diff --git a/parm/config/config.aeroanl b/parm/config/config.aeroanl index 3b9a9971f49..41d63f85490 100644 --- a/parm/config/config.aeroanl +++ b/parm/config/config.aeroanl @@ -18,7 +18,7 @@ export BERROR_DATE="20160630.000000" export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ -export JEDIVAREXE=${HOMEgfs}/exec/fv3jedi_var.x +export JEDIEXE=${HOMEgfs}/exec/fv3jedi_var.x export crtm_VERSION="2.3.0" echo "END: config.aeroanl" diff --git a/parm/config/config.atmanl b/parm/config/config.atmanl index 719018d1fdd..e7e2036d4fe 100644 --- a/parm/config/config.atmanl +++ b/parm/config/config.atmanl @@ -19,7 +19,7 @@ export layout_y=1 export io_layout_x=1 export io_layout_y=1 -export JEDIVAREXE=${HOMEgfs}/exec/fv3jedi_var.x +export JEDIEXE=${HOMEgfs}/exec/fv3jedi_var.x export crtm_VERSION="2.3.0" echo "END: config.atmanl" diff --git a/parm/config/config.atmensanl b/parm/config/config.atmensanl index ec03673027f..67c0250fc3e 100755 --- a/parm/config/config.atmensanl +++ b/parm/config/config.atmensanl @@ -17,7 +17,7 @@ export layout_y=1 export io_layout_x=1 export io_layout_y=1 -export JEDIENSEXE=${HOMEgfs}/exec/fv3jedi_letkf.x +export JEDIEXE=${HOMEgfs}/exec/fv3jedi_letkf.x export crtm_VERSION="2.3.0" echo "END: config.atmensanl" diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index e21284dc91c..8c692e22d67 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -68,7 +68,6 @@ def initialize(self: Analysis) -> None: - staging B error files - staging model backgrounds - generating a YAML file for the JEDI executable - - linking the JEDI executable (TODO make it copyable, requires JEDI fix) - creating output directories """ super().initialize() @@ -99,14 +98,6 @@ def initialize(self: Analysis) -> None: save_as_yaml(varda_yaml, self.task_config.fv3jedi_yaml) logger.info(f"Wrote variational YAML to: {self.task_config.fv3jedi_yaml}") - # link executable to DATA/ directory - exe_src = self.task_config['JEDIVAREXE'] - logger.debug(f"Link executable {exe_src} to DATA/") # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. - exe_dest = os.path.join(self.task_config['DATA'], os.path.basename(exe_src)) - if os.path.exists(exe_dest): - rm_p(exe_dest) - os.symlink(exe_src, exe_dest) - # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 6eeeb349965..94a93b74f47 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -5,7 +5,7 @@ from netCDF4 import Dataset from typing import List, Dict, Any -from pygw.yaml_file import YAMLFile, parse_j2yaml +from pygw.yaml_file import YAMLFile, parse_j2yaml, parse_yamltmpl from pygw.file_utils import FileHandler from pygw.template import Template, TemplateConstants from pygw.logger import logit @@ -36,6 +36,9 @@ def initialize(self) -> None: bias_dict = self.get_bias_dict() FileHandler(bias_dict).sync() + # link jedi executable to run directory + self.link_jediexe() + @logit(logger) def get_obs_dict(self: Task) -> Dict[str, Any]: """Compile a dictionary of observation files to copy @@ -171,3 +174,28 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: """ berror_dict = {'foo': 'bar'} return berror_dict + + @logit(logger) + def link_jediexe(self: Task) -> None: + """Compile a dictionary of background error files to copy + + This method links a JEDI executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + exe_src = self.task_config.JEDIEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.debug(f"Link executable {exe_src} to DATA/") + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + return diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index a632e318d95..045839edfd7 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -67,7 +67,6 @@ def initialize(self: Analysis) -> None: - staging B error files - staging model backgrounds - generating a YAML file for the JEDI executable - - linking the JEDI executable (TODO make it copyable, requires JEDI fix) - creating output directories """ super().initialize() @@ -98,14 +97,6 @@ def initialize(self: Analysis) -> None: save_as_yaml(varda_yaml, self.task_config.fv3jedi_yaml) logger.info(f"Wrote variational YAML to: {self.task_config.fv3jedi_yaml}") - # link executable to DATA/ directory - exe_src = self.task_config.JEDIVAREXE - logger.debug(f"Link executable {exe_src} to DATA/") # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. - exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) - if os.path.exists(exe_dest): - rm_p(exe_dest) - os.symlink(exe_src, exe_dest) - # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 86563658207..097f14cdf4f 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -67,7 +67,6 @@ def initialize(self: Analysis) -> None: - staging FV3-JEDI fix files - staging model backgrounds - generating a YAML file for the JEDI executable - - linking the JEDI executable (TODO make it copyable, requires JEDI fix) - creating output directories Parameters @@ -101,14 +100,6 @@ def initialize(self: Analysis) -> None: save_as_yaml(ensda_yaml, self.task_config.fv3jedi_yaml) logger.info(f"Wrote ensemble da YAML to: {self.task_config.fv3jedi_yaml}") - # link executable to DATA/ directory - exe_src = self.task_config.JEDIENSEXE - logger.debug(f"Link executable {exe_src} to DATA/") # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. - exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) - if os.path.exists(exe_dest): - rm_p(exe_dest) - os.symlink(exe_src, exe_dest) - # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ From ddfc81164482fb7125e42eadab2101117f368513 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 20 Apr 2023 10:49:36 +0000 Subject: [PATCH 21/25] correctly set cycledefs for gfs atmanlinit (#1313) --- workflow/rocoto/workflow_tasks.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py index 27ffa8e5531..891d3f5c6d3 100644 --- a/workflow/rocoto/workflow_tasks.py +++ b/workflow/rocoto/workflow_tasks.py @@ -397,6 +397,7 @@ def analdiag(self): return task def atmanlinit(self): + deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} deps.append(rocoto.add_dependency(dep_dict)) @@ -407,7 +408,13 @@ def atmanlinit(self): else: dependencies = rocoto.create_dependency(dep=deps) - cycledef = "gdas" + gfs_cyc = self._base["gfs_cyc"] + gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_cdumps else False + + cycledef = self.cdump + if self.cdump in ['gfs'] and gfs_enkf and gfs_cyc != 4: + cycledef = 'gdas' + resources = self.get_resource('atmanlinit') task = create_wf_task('atmanlinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, cycledef=cycledef) From 30db276ca20e8887ee686edb7722496508a63a42 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 20 Apr 2023 20:00:26 +0000 Subject: [PATCH 22/25] remove FV3JEDI_FIX from atmanl and atmensanl config files (#1313) --- parm/config/config.atmanl | 1 - parm/config/config.atmensanl | 1 - 2 files changed, 2 deletions(-) diff --git a/parm/config/config.atmanl b/parm/config/config.atmanl index e7e2036d4fe..c0cd9e6733d 100644 --- a/parm/config/config.atmanl +++ b/parm/config/config.atmanl @@ -10,7 +10,6 @@ export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/atm/obs/lists/gdas_prototype_3d.yam export ATMVARYAML=${HOMEgfs}/sorc/gdas.cd/parm/atm/variational/3dvar_dripcg.yaml export STATICB_TYPE="gsibec" export BERROR_YAML=${HOMEgfs}/sorc/gdas.cd/parm/atm/berror/staticb_${STATICB_TYPE}.yaml -export FV3JEDI_FIX=${HOMEgfs}/fix/gdas export INTERP_METHOD='barycentric' export layout_x=1 diff --git a/parm/config/config.atmensanl b/parm/config/config.atmensanl index 67c0250fc3e..4d945ea717e 100755 --- a/parm/config/config.atmensanl +++ b/parm/config/config.atmensanl @@ -8,7 +8,6 @@ echo "BEGIN: config.atmensanl" export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/atm/obs/config/ export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/atm/obs/lists/lgetkf_prototype.yaml export ATMENSYAML=${HOMEgfs}/sorc/gdas.cd/parm/atm/lgetkf/lgetkf.yaml -export FV3JEDI_FIX=${HOMEgfs}/fix/gdas export INTERP_METHOD='barycentric' export layout_x=1 From 8056bf9836ac3d8dee9c2aca453e2f671670d6db Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 20 Apr 2023 20:12:59 +0000 Subject: [PATCH 23/25] streamline creation of ensemble member analysis directories (#1313) --- ush/python/pygfs/task/atmens_analysis.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 097f14cdf4f..1a5f22406b7 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -109,12 +109,11 @@ def initialize(self: Analysis) -> None: FileHandler({'mkdir': newdirs}).sync() # Make directories for member analysis files + anldir = [] for imem in range(1, self.task_config.NMEM_ENKF + 1): memchar = f"mem{imem:03d}" - anldir = [ - os.path.join(self.task_config.DATA, 'anl', memchar) - ] - FileHandler({'mkdir': anldir}).sync() + anldir.append(os.path.join(self.task_config.DATA, 'anl', f'mem{imem:03d}')) + FileHandler({'mkdir': anldir}).sync() @logit(logger) def execute(self: Analysis) -> None: From 812cb466b1441c1418a7541aeec957a03050ee20 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 20 Apr 2023 23:37:15 +0000 Subject: [PATCH 24/25] add TODO for MPMD or parallelize with mpi4py in atmens_analysis.py (#1313) --- ush/python/pygfs/task/atmens_analysis.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 1a5f22406b7..c781ce42280 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -302,6 +302,8 @@ def jedi2fv3inc(self: Analysis) -> None: f"{self.task_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atminc.nc") # Execute incpy to create the UFS model atm increment file + # TODO: use MPMD or parallelize with mpi4py. + # See https://github.com/NOAA-EMC/global-workflow/pull/1373#discussion_r1173060656 cmd = Executable(incpy) cmd.add_default_arg(atmges_fv3) cmd.add_default_arg(atminc_jedi) From 69624453ebbb5d9fcdcd926e48a5c842b82d79f1 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 20 Apr 2023 23:41:19 +0000 Subject: [PATCH 25/25] remove trailing whitespace in atmens_analysis.py comment (#1313) --- ush/python/pygfs/task/atmens_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index c781ce42280..636129d3ee5 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -302,7 +302,7 @@ def jedi2fv3inc(self: Analysis) -> None: f"{self.task_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atminc.nc") # Execute incpy to create the UFS model atm increment file - # TODO: use MPMD or parallelize with mpi4py. + # TODO: use MPMD or parallelize with mpi4py # See https://github.com/NOAA-EMC/global-workflow/pull/1373#discussion_r1173060656 cmd = Executable(incpy) cmd.add_default_arg(atmges_fv3)