From a2f78d55a4501ef5b88f7ce73d12cad247bcce84 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Sat, 25 Feb 2023 19:40:48 +0000 Subject: [PATCH 01/13] add sondes to test_gdasapp_atm_jjob_var_run, replace geometric height with geopoential height in sondes.yaml (#357) --- parm/atm/obs/config/sondes.yaml | 4 ++-- test/atm/global-workflow/gdas_prototype.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/atm/obs/config/sondes.yaml b/parm/atm/obs/config/sondes.yaml index b1b6cf9e9..3131a1bf4 100644 --- a/parm/atm/obs/config/sondes.yaml +++ b/parm/atm/obs/config/sondes.yaml @@ -26,7 +26,7 @@ obs operator: - name: specificHumidity - name: SfcPCorrected da_psfc_scheme: GSI - geovar_sfc_geomz: surface_geometric_height + geovar_sfc_geomz: surface_geopotential_height geovar_geomz: geopotential_height variables: - name: stationPressure @@ -92,7 +92,7 @@ obs filters: error_min: 100.0 # 1 mb error_max: 300.0 # 3 mb geovar_geomz: geopotential_height - geovar_sfc_geomz: surface_geometric_height + geovar_sfc_geomz: surface_geopotential_height # # Gross error check with (O - B) / ObsError greater than threshold - filter: Background Check diff --git a/test/atm/global-workflow/gdas_prototype.yaml b/test/atm/global-workflow/gdas_prototype.yaml index fd867bcd0..18d64e54e 100644 --- a/test/atm/global-workflow/gdas_prototype.yaml +++ b/test/atm/global-workflow/gdas_prototype.yaml @@ -1,6 +1,6 @@ observers: - !INC ${OBS_YAML_DIR}/amsua_n19.yaml -##- !INC ${OBS_YAML_DIR}/sondes.yaml +- !INC ${OBS_YAML_DIR}/sondes.yaml ##- !INC ${OBS_YAML_DIR}/atms_n20.yaml ##- !INC ${OBS_YAML_DIR}/aircraft.yaml ##- !INC ${OBS_YAML_DIR}/satwind.yaml From 38cae2c3c091fa1cb5f287a7898cb8d3c66080f6 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 10 Mar 2023 11:06:05 +0000 Subject: [PATCH 02/13] update atm_jjob var and ens ctests to work with python UFS-DA ATM g-w (#357) --- test/atm/global-workflow/CMakeLists.txt | 16 ++++++------ .../{config.atmanal => config.atmanl} | 12 ++++++--- test/atm/global-workflow/config.base.emc.dyn | 12 ++++++--- test/atm/global-workflow/config.yaml | 3 ++- .../{jjob_var_post.sh => jjob_ens_final.sh} | 14 +++++++--- .../{jjob_ens_prep.sh => jjob_ens_init.sh} | 26 ++++++++++++++----- test/atm/global-workflow/jjob_ens_run.sh | 12 ++++++--- .../{jjob_ens_post.sh => jjob_var_final.sh} | 12 ++++++--- .../{jjob_var_prep.sh => jjob_var_init.sh} | 12 ++++++--- test/atm/global-workflow/jjob_var_run.sh | 10 +++++-- .../atm/global-workflow/setup_workflow_exp.sh | 7 ++++- 11 files changed, 97 insertions(+), 39 deletions(-) rename test/atm/global-workflow/{config.atmanal => config.atmanl} (76%) rename test/atm/global-workflow/{jjob_var_post.sh => jjob_ens_final.sh} (65%) rename test/atm/global-workflow/{jjob_ens_prep.sh => jjob_ens_init.sh} (61%) rename test/atm/global-workflow/{jjob_ens_post.sh => jjob_var_final.sh} (70%) rename test/atm/global-workflow/{jjob_var_prep.sh => jjob_var_init.sh} (75%) diff --git a/test/atm/global-workflow/CMakeLists.txt b/test/atm/global-workflow/CMakeLists.txt index c31571c58..8a1708a8f 100644 --- a/test/atm/global-workflow/CMakeLists.txt +++ b/test/atm/global-workflow/CMakeLists.txt @@ -6,8 +6,8 @@ add_test(NAME test_gdasapp_setup_atm_cycled_exp ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -add_test(NAME test_gdasapp_atm_jjob_var_prep - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_prep.sh +add_test(NAME test_gdasapp_atm_jjob_var_init + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_init.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) @@ -16,13 +16,13 @@ add_test(NAME test_gdasapp_atm_jjob_var_run ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -add_test(NAME test_gdasapp_atm_jjob_var_post - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_post.sh +add_test(NAME test_gdasapp_atm_jjob_var_final + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_final.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -add_test(NAME test_gdasapp_atm_jjob_ens_prep - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_prep.sh +add_test(NAME test_gdasapp_atm_jjob_ens_init + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_init.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) @@ -31,7 +31,7 @@ add_test(NAME test_gdasapp_atm_jjob_ens_run ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -add_test(NAME test_gdasapp_atm_jjob_ens_post - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_post.sh +add_test(NAME test_gdasapp_atm_jjob_ens_final + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_final.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) diff --git a/test/atm/global-workflow/config.atmanal b/test/atm/global-workflow/config.atmanl similarity index 76% rename from test/atm/global-workflow/config.atmanal rename to test/atm/global-workflow/config.atmanl index 0ee19fb54..edb473045 100755 --- a/test/atm/global-workflow/config.atmanal +++ b/test/atm/global-workflow/config.atmanl @@ -1,9 +1,9 @@ #!/bin/bash -x -########## config.atmanal ########## +########## config.atmanl ########## # configuration common to all atm analysis tasks -echo "BEGIN: config.atmanal" +echo "BEGIN: config.atmanl" export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/atm/obs/config/ export OBS_LIST=@OBS_LIST@ @@ -19,7 +19,13 @@ export R2D2_BC_DUMP='oper_gdas' export R2D2_ARCH_DB='local' export INTERP_METHOD='barycentric' +export layout_x=1 +export layout_y=1 + export io_layout_x=1 export io_layout_y=1 -echo "END: config.atmanal" +export JEDIVAREXE=${HOMEgfs}/exec/fv3jedi_var.x +export crtm_VERSION="2.3.0" + +echo "END: config.atmanl" diff --git a/test/atm/global-workflow/config.base.emc.dyn b/test/atm/global-workflow/config.base.emc.dyn index 47d8bbc96..1c7239931 100755 --- a/test/atm/global-workflow/config.base.emc.dyn +++ b/test/atm/global-workflow/config.base.emc.dyn @@ -40,7 +40,7 @@ export FIXreg2grb2=$HOMEgfs/fix/reg2grb2 export PACKAGEROOT="@PACKAGEROOT@" # TODO: set via prod_envir in Ops export COMROOT="@COMROOT@" # TODO: set via prod_envir in Ops export COMINsyn="@COMINsyn@" -export DMPDIR="@DMPDIR@" +export DMPDIR=@DUMPDIR@ # USER specific paths export HOMEDIR="@HOMEDIR@" @@ -109,7 +109,7 @@ export DUMP_SUFFIX="" if [[ "$CDATE" -ge "2019092100" && "$CDATE" -le "2019110700" ]]; then export DUMP_SUFFIX="p" # Use dumps from NCO GFS v15.3 parallel fi -export DATAROOT="@DATAPATH@/RUNDIR/${jobid}" # TODO: set via prod_envir in Ops +export DATAROOT="@DATAPATH@/RUNDIR" # TODO: set via prod_envir in Ops export RUNDIR="${DATAROOT}" # TODO: Should be removed; use DATAROOT instead export ARCDIR="$NOSCRUB/archive/$PSLOT" export ICSDIR="@ICSDIR@" @@ -119,8 +119,12 @@ export ATARDIR="@ATARDIR@" export envir=${envir:-"prod"} export NET="gfs" export RUN=${RUN:-${CDUMP:-"gfs"}} -export COMIN_OBS=${DMPDIR}/${CDUMP}.${PDY}/$cyc/atmos -export COMIN_GES_OBS=${DMPDIR}/${CDUMP}.${PDY}/$cyc/atmos +CDUMP_OBS="${CDUMP}" +if [ "${CDUMP_OBS}" = "enkfgdas" ]; then + CDUMP_OBS="gdas" +fi +export COMIN_OBS=${DMPDIR}/${CDUMP_OBS}.${PDY}/$cyc/atmos +export COMIN_GES_OBS=${DMPDIR}/${CDUMP_OBS}.${PDY}/$cyc/atmos export COMINatmos=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/atmos export COMOUTatmos=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/atmos export COMINwave=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/wave diff --git a/test/atm/global-workflow/config.yaml b/test/atm/global-workflow/config.yaml index b406303a7..2a70597a7 100644 --- a/test/atm/global-workflow/config.yaml +++ b/test/atm/global-workflow/config.yaml @@ -4,5 +4,6 @@ base: DO_JEDIENS: "YES" UTILROOT: /apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2 DATAPATH: "@bindir@/test/atm/global-workflow/testrun" -atmanal: + DUMPDIR: @dumpdir@ +atmanl: OBS_LIST: "@srcdir@/test/atm/global-workflow/gdas_prototype.yaml" diff --git a/test/atm/global-workflow/jjob_var_post.sh b/test/atm/global-workflow/jjob_ens_final.sh similarity index 65% rename from test/atm/global-workflow/jjob_var_post.sh rename to test/atm/global-workflow/jjob_ens_final.sh index 815220e88..489f25a2f 100755 --- a/test/atm/global-workflow/jjob_var_post.sh +++ b/test/atm/global-workflow/jjob_ens_final.sh @@ -12,19 +12,27 @@ export PDY=20210323 export cyc=18 export CDATE=${PDY}${cyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS -export CDUMP=gdas +export RUN=enkfgdas +export CDUMP=enkfgdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} export jobid=$pid export COMROOT=$DATAROOT +export NMEM_ENKF=3 export ACCOUNT=da-cpu +export DO_JEDIENS="YES" + +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_POST + ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINAL else - sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_POST + sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_prep.sh b/test/atm/global-workflow/jjob_ens_init.sh similarity index 61% rename from test/atm/global-workflow/jjob_ens_prep.sh rename to test/atm/global-workflow/jjob_ens_init.sh index b1b0a3c3f..fd27980be 100755 --- a/test/atm/global-workflow/jjob_ens_prep.sh +++ b/test/atm/global-workflow/jjob_ens_init.sh @@ -12,7 +12,8 @@ export PDY=20210323 export cyc=18 export CDATE=${PDY}${cyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS -export CDUMP=gdas +export RUN=enkfgdas +export CDUMP=enkfgdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} @@ -21,6 +22,11 @@ export COMROOT=$DATAROOT export NMEM_ENKF=3 export ACCOUNT=da-cpu +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') @@ -28,23 +34,29 @@ machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"' GDATE=`date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - 6 hours"` gPDY=$(echo $GDATE | cut -c1-8) gcyc=$(echo $GDATE | cut -c9-10) -mkdir -p $ROTDIR/gdas_test/enkfgdas.$gPDY/$gcyc/atmos/ + +mkdir -p $ROTDIR/gdas_test/gdas.$gPDY/$gcyc/atmos/ +flist="satbias tlapse" +for file in $flist; do + cp -r $GDASAPP_TESTDATA/lowres/gdas.$gPDY/$gcyc/atmos/*${file}* $ROTDIR/gdas_test/gdas.$gPDY/$gcyc/atmos/ +done # Copy tiled ges and atmf006 files to ROTDIR +mkdir -p $ROTDIR/gdas_test/enkfgdas.$gPDY/$gcyc/atmos/ for imem in $(seq 1 $NMEM_ENKF); do memchar="mem"$(printf %03i $imem) - source=$GDASAPP_TESTDATA/lowres/enkfgdas.$gPDY/$gcyc/atmos/$memchar - target=$ROTDIR/gdas_test/enkfgdas.$gPDY/$gcyc/atmos/$memchar + source=$GDASAPP_TESTDATA/lowres/enkfgdas.$gPDY/$gcyc/$memchar/atmos + target=$ROTDIR/gdas_test/enkfgdas.$gPDY/$gcyc/$memchar/atmos mkdir -p $target rm -rf $target/RESTART ln -fs $source/RESTART $target/ - ln -fs $source/gdas.t${gcyc}z.atmf006.nc $target/ + ln -fs $source/enkfgdas.t${gcyc}z.atmf006.nc $target/ done if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP + ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE else - sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP + sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_run.sh b/test/atm/global-workflow/jjob_ens_run.sh index 35f7d5ec7..d3a5899ab 100755 --- a/test/atm/global-workflow/jjob_ens_run.sh +++ b/test/atm/global-workflow/jjob_ens_run.sh @@ -15,7 +15,8 @@ export gPDY=20210323 export gcyc=12 export GDATE=${gPDY}${gcyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS -export CDUMP=gdas +export RUN=enkfgdas +export CDUMP=enkfgdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} @@ -25,13 +26,18 @@ export NMEM_ENKF=3 export ACCOUNT=da-cpu export DO_JEDIENS="YES" +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') if [ $machine = 'HERA' -o $machine = 'ORION' ]; then - sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=debug --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN + sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=debug --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN else - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN + ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN fi diff --git a/test/atm/global-workflow/jjob_ens_post.sh b/test/atm/global-workflow/jjob_var_final.sh similarity index 70% rename from test/atm/global-workflow/jjob_ens_post.sh rename to test/atm/global-workflow/jjob_var_final.sh index 06c8b81b3..c3e61e7a0 100755 --- a/test/atm/global-workflow/jjob_ens_post.sh +++ b/test/atm/global-workflow/jjob_var_final.sh @@ -12,21 +12,25 @@ export PDY=20210323 export cyc=18 export CDATE=${PDY}${cyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS +export RUN=gdas export CDUMP=gdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} export jobid=$pid export COMROOT=$DATAROOT -export NMEM_ENKF=3 export ACCOUNT=da-cpu -export DO_JEDIENS="YES" + +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST + ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE else - sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST + sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE fi diff --git a/test/atm/global-workflow/jjob_var_prep.sh b/test/atm/global-workflow/jjob_var_init.sh similarity index 75% rename from test/atm/global-workflow/jjob_var_prep.sh rename to test/atm/global-workflow/jjob_var_init.sh index c4d2a89ee..c0b875502 100755 --- a/test/atm/global-workflow/jjob_var_prep.sh +++ b/test/atm/global-workflow/jjob_var_init.sh @@ -12,6 +12,7 @@ export PDY=20210323 export cyc=18 export CDATE=${PDY}${cyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS +export RUN=gdas export CDUMP=gdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg @@ -20,6 +21,11 @@ export jobid=$pid export COMROOT=$DATAROOT export ACCOUNT=da-cpu +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') @@ -29,13 +35,13 @@ gPDY=$(echo $GDATE | cut -c1-8) gcyc=$(echo $GDATE | cut -c9-10) mkdir -p $ROTDIR/gdas_test/gdas.$gPDY/$gcyc/atmos/ -flist="abias atmf006 RESTART" +flist="abias atmf006 satbias tlapse RESTART" for file in $flist; do cp -r $GDASAPP_TESTDATA/lowres/gdas.$gPDY/$gcyc/atmos/*${file}* $ROTDIR/gdas_test/gdas.$gPDY/$gcyc/atmos/ done if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_PREP + ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE else - sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_PREP + sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_var_run.sh b/test/atm/global-workflow/jjob_var_run.sh index 30da673b8..eb44fa707 100755 --- a/test/atm/global-workflow/jjob_var_run.sh +++ b/test/atm/global-workflow/jjob_var_run.sh @@ -15,6 +15,7 @@ export gPDY=20210323 export gcyc=12 export GDATE=${gPDY}${gcyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS +export RUN=gdas export CDUMP=gdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg @@ -23,11 +24,16 @@ export jobid=$pid export COMROOT=$DATAROOT export ACCOUNT=da-cpu +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_RUN + ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN else - sbatch -n 6 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_RUN + sbatch -n 6 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN fi diff --git a/test/atm/global-workflow/setup_workflow_exp.sh b/test/atm/global-workflow/setup_workflow_exp.sh index bedd46ab8..8d18fc6e5 100755 --- a/test/atm/global-workflow/setup_workflow_exp.sh +++ b/test/atm/global-workflow/setup_workflow_exp.sh @@ -24,12 +24,13 @@ rm -rf $comrot $expdir config # copy config.yaml to local config cp -r $configdir config cp $srcdir/test/atm/global-workflow/config.base.emc.dyn config/ -cp $srcdir/test/atm/global-workflow/config.atmanal config/ +cp $srcdir/test/atm/global-workflow/config.atmanl config/ cp $srcdir/test/atm/global-workflow/config.yaml . # update paths in config.yaml sed -i -e "s~@bindir@~${bindir}~g" config.yaml sed -i -e "s~@srcdir@~${srcdir}~g" config.yaml +sed -i -e "s~@dumpdir@~${GDASAPP_TESTDATA}/lowres~g" config.yaml # run the script ln -sf $srcdir/../../workflow/setup_expt.py . @@ -50,4 +51,8 @@ $srcdir/../../workflow/setup_expt.py cycled --idate $idate \ --expdir $expdir \ --yaml $expdir/../config.yaml +echo " " +echo "$expdir/../config.yaml is" +cat $expdir/../config.yaml + exit $? From e58fa6b528978854ceb4741648dd897d6337a689 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 10 Mar 2023 11:35:33 +0000 Subject: [PATCH 03/13] correct yaml syntax error in config.yaml (#357) --- test/atm/global-workflow/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/atm/global-workflow/config.yaml b/test/atm/global-workflow/config.yaml index 2a70597a7..f3c699f8b 100644 --- a/test/atm/global-workflow/config.yaml +++ b/test/atm/global-workflow/config.yaml @@ -4,6 +4,6 @@ base: DO_JEDIENS: "YES" UTILROOT: /apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2 DATAPATH: "@bindir@/test/atm/global-workflow/testrun" - DUMPDIR: @dumpdir@ + DUMPDIR: "@dumpdir@" atmanl: OBS_LIST: "@srcdir@/test/atm/global-workflow/gdas_prototype.yaml" From b3a7013912e84ee411607da29fd3840fd7097b3c Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Sat, 11 Mar 2023 00:19:04 +0000 Subject: [PATCH 04/13] correct name of ens final j-job on Orion (#357) --- test/atm/global-workflow/jjob_ens_final.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/atm/global-workflow/jjob_ens_final.sh b/test/atm/global-workflow/jjob_ens_final.sh index 489f25a2f..b1197631d 100755 --- a/test/atm/global-workflow/jjob_ens_final.sh +++ b/test/atm/global-workflow/jjob_ens_final.sh @@ -32,7 +32,7 @@ export PYTHONPATH machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINAL + ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE else sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE fi From e03ebb16c2fbcef2982cbc6fc5bed65df0c3ccc3 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 21 Mar 2023 17:46:11 +0000 Subject: [PATCH 05/13] lgetkf updates: specify analysis variables, turn off posterior observer (#357) --- parm/atm/lgetkf/lgetkf.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parm/atm/lgetkf/lgetkf.yaml b/parm/atm/lgetkf/lgetkf.yaml index 1869c15d9..f94999515 100644 --- a/parm/atm/lgetkf/lgetkf.yaml +++ b/parm/atm/lgetkf/lgetkf.yaml @@ -12,6 +12,7 @@ geometry: field metadata override: ./fv3jedi/gfs-restart.yaml window begin: '$(ATM_WINDOW_BEGIN)' window length: PT6H +increment variables: [ua,va,DZ,delp,t,ps,sphum,ice_wat,liq_wat,o3mr] background: datetime: '{{BKG_ISOTIME}}' @@ -41,6 +42,7 @@ driver: save prior mean: false save posterior mean increment: false save posterior ensemble increments: true + do posterior observer: false local ensemble DA: solver: GETKF From db2506567e70ad0b4c2395a67d47df3e2173a180 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 30 Mar 2023 23:30:17 +0000 Subject: [PATCH 06/13] update yamls used in UFS-DA ATM prototype cycling to be consistent with g-w python (#357) --- parm/atm/lgetkf/lgetkf.yaml | 24 +++++++++--------- parm/atm/obs/config/amsua_n19.yaml | 16 ++++++------ parm/atm/obs/config/lgetkf_amsua_n19.yaml | 14 +++++------ parm/atm/obs/config/lgetkf_sondes.yaml | 4 +-- parm/atm/obs/config/sondes.yaml | 4 +-- parm/atm/variational/3dvar_dripcg.yaml | 30 +++++++++++------------ 6 files changed, 46 insertions(+), 46 deletions(-) diff --git a/parm/atm/lgetkf/lgetkf.yaml b/parm/atm/lgetkf/lgetkf.yaml index f94999515..13f8d1766 100644 --- a/parm/atm/lgetkf/lgetkf.yaml +++ b/parm/atm/lgetkf/lgetkf.yaml @@ -4,33 +4,33 @@ geometry: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) field metadata override: ./fv3jedi/gfs-restart.yaml -window begin: '$(ATM_WINDOW_BEGIN)' -window length: PT6H +window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' +window length: $(ATM_WINDOW_LENGTH) increment variables: [ua,va,DZ,delp,t,ps,sphum,ice_wat,liq_wat,o3mr] background: - datetime: '{{BKG_ISOTIME}}' + datetime: '{{ current_cycle | to_isotime }}' members from template: template: - datetime: '{{BKG_ISOTIME}}' + datetime: '{{ current_cycle | to_isotime }}' filetype: fms restart state variables: [ua,va,t,DZ,delp,ps,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] datapath: bkg/mem%mem%/RESTART - filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res' + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' pattern: '%mem%' - nmembers: !ENV ${NMEM_ENKF} + nmembers: $(NMEM_ENKF) zero padding: 3 observations: !INC ${OBS_LIST} diff --git a/parm/atm/obs/config/amsua_n19.yaml b/parm/atm/obs/config/amsua_n19.yaml index 0096f2fd0..f22eecdb8 100644 --- a/parm/atm/obs/config/amsua_n19.yaml +++ b/parm/atm/obs/config/amsua_n19.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}amsua_n19.${PDY}${cyc}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)amsua_n19.{{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_amsua_n19_${PDY}${cyc}.nc4 + obsfile: $(DATA)/diags/diag_amsua_n19_{{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightnessTemperature] @@ -20,16 +20,16 @@ obs operator: obs options: Sensor_ID: amsua_n19 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias.${gPDY}${gcyc}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias.${PDY}${cyc}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias.{{ previous_cycle | to_YMDH }}.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias.{{ current_cycle | to_YMDH }}.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &amsua_n19_tlapse !ENV ${DATA}/obs/${GPREFIX}amsua_n19.tlapse.${gPDY}${gcyc}.txt + tlapse: &amsua_n19_tlapse $(DATA)/obs/$(GPREFIX)amsua_n19.tlapse.{{ previous_cycle | to_YMDH }}.txt - name: lapse_rate tlapse: *amsua_n19_tlapse - name: emissivity @@ -46,11 +46,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias_cov.${gPDY}${gcyc}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias_cov.{{ previous_cycle | to_YMDH }}.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias_cov.${PDY}${cyc}.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias_cov.{{ current_cycle | to_YMDH }}.nc4 obs filters: - filter: BlackList filter variables: diff --git a/parm/atm/obs/config/lgetkf_amsua_n19.yaml b/parm/atm/obs/config/lgetkf_amsua_n19.yaml index 77dff58ae..78fd790f7 100644 --- a/parm/atm/obs/config/lgetkf_amsua_n19.yaml +++ b/parm/atm/obs/config/lgetkf_amsua_n19.yaml @@ -6,11 +6,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ./obs/${OPREFIX}amsua_n19.${PDY}${cyc}.nc4 + obsfile: ./obs/$(OPREFIX)amsua_n19.{{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ./diags/diag_amsua_n19_${PDY}${cyc}.nc4 + obsfile: ./diags/diag_amsua_n19_{{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightnessTemperature] @@ -27,14 +27,14 @@ obs operator: obs error: covariance model: diagonal obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias.${gPDY}${gcyc}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias.${PDY}${cyc}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias.{{ previous_cycle | to_YMDH }}.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias.{{ current_cycle | to_YMDH }}.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &amsua_n19_tlapse !ENV ${DATA}/obs/${GPREFIX}amsua_n19.tlapse.${gPDY}${gcyc}.txt + tlapse: &amsua_n19_tlapse $(DATA)/obs/$(GPREFIX)amsua_n19.tlapse.{{ previous_cycle | to_YMDH }}.txt - name: lapse_rate tlapse: *amsua_n19_tlapse - name: emissivity @@ -51,11 +51,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias_cov.${gPDY}${gcyc}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias_cov.{{ previous_cycle | to_YMDH }}.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias_cov.${PDY}${cyc}.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias_cov.{{ current_cycle | to_YMDH }}.nc4 obs filters: - filter: Bounds Check filter variables: diff --git a/parm/atm/obs/config/lgetkf_sondes.yaml b/parm/atm/obs/config/lgetkf_sondes.yaml index 837b54a48..4f2358097 100644 --- a/parm/atm/obs/config/lgetkf_sondes.yaml +++ b/parm/atm/obs/config/lgetkf_sondes.yaml @@ -6,11 +6,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ./obs/${OPREFIX}sondes.${PDY}${cyc}.nc4 + obsfile: ./obs/$(OPREFIX)sondes.{{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ./diags/diag_sondes_lgetkf_${PDY}${cyc}.nc4 + obsfile: ./diags/diag_sondes_lgetkf_{{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [windEastward, windNorthward, airTemperature] diff --git a/parm/atm/obs/config/sondes.yaml b/parm/atm/obs/config/sondes.yaml index 3131a1bf4..e9331f7bd 100644 --- a/parm/atm/obs/config/sondes.yaml +++ b/parm/atm/obs/config/sondes.yaml @@ -3,7 +3,7 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}sondes.${PDY}${cyc}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)sondes.{{ current_cycle | to_YMDH }}.nc4 obsgrouping: group variables: ["stationIdentification"] sort variable: "pressure" @@ -11,7 +11,7 @@ obs space: obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_sondes_${PDY}${cyc}.nc4 + obsfile: $(DATA)/diags/diag_sondes_{{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [stationPressure, airTemperature, windEastward, windNorthward, specificHumidity] diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml index 2ea6542b0..4563f45ac 100644 --- a/parm/atm/variational/3dvar_dripcg.yaml +++ b/parm/atm/variational/3dvar_dripcg.yaml @@ -1,6 +1,6 @@ cost function: cost type: 3D-Var - window begin: '{{ATM_WINDOW_BEGIN}}' + window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' window length: $(ATM_WINDOW_LENGTH) analysis variables: &3dvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr] geometry: @@ -9,8 +9,8 @@ cost function: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) @@ -18,12 +18,12 @@ cost function: background: datapath: ./bkg filetype: fms restart - datetime: '{{BKG_ISOTIME}}' - filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res' + datetime: '{{ current_cycle | to_isotime }}' + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' state variables: [ua,va,t,DZ,delp,ps,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] @@ -42,8 +42,8 @@ variational: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_anl) npy: $(npy_anl) npz: $(npz_anl) @@ -59,8 +59,8 @@ variational: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_anl) npy: $(npy_anl) npz: $(npz_anl) @@ -81,8 +81,8 @@ final: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_anl) npy: $(npy_anl) npz: $(npz_anl) From 1ad6af012aa12665a3972ed573c4ccd3b6aa1e9d Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 5 Apr 2023 17:54:58 +0000 Subject: [PATCH 07/13] remove RESTART from bkg path --- parm/atm/lgetkf/lgetkf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/atm/lgetkf/lgetkf.yaml b/parm/atm/lgetkf/lgetkf.yaml index 13f8d1766..b8a6f9886 100644 --- a/parm/atm/lgetkf/lgetkf.yaml +++ b/parm/atm/lgetkf/lgetkf.yaml @@ -23,7 +23,7 @@ background: state variables: [ua,va,t,DZ,delp,ps,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] - datapath: bkg/mem%mem%/RESTART + datapath: bkg/mem%mem% filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' From a3ca40fdd11bbb71bd51c48db239c0984618d3f4 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Sat, 8 Apr 2023 19:38:47 +0000 Subject: [PATCH 08/13] Update UFS DA ATM yaml and scripts to use g-w pygw (#357) --- parm/atm/berror/hybvar_bump.yaml | 16 +++++----- parm/atm/berror/hybvar_gsibec.yaml | 22 +++++++------- parm/atm/berror/staticb_bump.yaml | 2 +- parm/atm/berror/staticb_gsibec.yaml | 8 ++--- parm/atm/hofx/hofx4d.yaml | 32 +++++++++---------- parm/atm/hofx/hofx_nomodel.yaml | 28 ++++++++--------- parm/atm/obs/config/aircraft.yaml | 4 +-- parm/atm/obs/config/atms_n20.yaml | 16 +++++----- parm/atm/obs/config/cris-fsr_n20.yaml | 16 +++++----- parm/atm/obs/config/cris-fsr_npp.yaml | 16 +++++----- parm/atm/obs/config/omi_aura.yaml | 4 +-- parm/atm/obs/config/ompsnp_npp.yaml | 4 +-- parm/atm/obs/config/ompstc8_npp.yaml | 4 +-- parm/atm/obs/config/satwind.yaml | 4 +-- parm/atm/obs/config/sfc.yaml | 4 +-- parm/atm/obs/config/sfcship.yaml | 4 +-- parm/atm/obs/config/ssmis_f17.yaml | 12 ++++---- parm/atm/obs/config/ssmis_f18.yaml | 12 ++++---- test/soca/CMakeLists.txt | 2 +- ush/run_jedi_exe.py | 44 ++++++++++++++++++++++++--- ush/ufsda/stage.py | 32 +++++++++++-------- 21 files changed, 164 insertions(+), 122 deletions(-) diff --git a/parm/atm/berror/hybvar_bump.yaml b/parm/atm/berror/hybvar_bump.yaml index ba39a78c0..879441e46 100644 --- a/parm/atm/berror/hybvar_bump.yaml +++ b/parm/atm/berror/hybvar_bump.yaml @@ -10,7 +10,7 @@ components: active variables: &active_vars [psi,chi,t,ps,rh,liq_wat,o3mr] bump: io: - data directory: !ENV &staticb_dir ${DATA}/berror + data directory: &staticb_dir $(DATA)/berror drivers: multivariate strategy: specific_univariate read local nicas: true @@ -95,17 +95,17 @@ components: covariance model: ensemble members from template: template: - datetime: '{{BKG_ISOTIME}}' + datetime: '{{ current_cycle | to_isotime }}' filetype: fms restart state variables: *3dvars datapath: ens/mem%mem%/ - filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res' + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' pattern: '%mem%' - nmembers: !ENV ${NMEM_ENKF} + nmembers: $(NMEM_ENKF) zero padding: 3 # localization: # localization method: SABER diff --git a/parm/atm/berror/hybvar_gsibec.yaml b/parm/atm/berror/hybvar_gsibec.yaml index ce381d079..33df8ca16 100644 --- a/parm/atm/berror/hybvar_gsibec.yaml +++ b/parm/atm/berror/hybvar_gsibec.yaml @@ -8,10 +8,10 @@ components: specific_humidity,cloud_liquid_ice,cloud_liquid_water, mole_fraction_of_ozone_in_air] output variables: *control_vars - gsi akbk: !ENV &akbk ${DATA}/fv3jedi/akbk.nc4 - gsi error covariance file: !ENV &gsiberr ${DATA}/berror/gsi-coeffs-gfs-global.nc4 -# gsi error covariance file: !ENV &gsiberr ${DATA}/berror/global_berror.f77 - gsi berror namelist file: !ENV &gsibnml ${DATA}/berror/gfs_gsi_global.nml + gsi akbk: &akbk $(DATA)/fv3jedi/akbk.nc4 + gsi error covariance file: &gsiberr $(DATA)/berror/gsi-coeffs-gfs-global.nc4 +# gsi error covariance file: &gsiberr $(DATA)/berror/global_berror.f77 + gsi berror namelist file: &gsibnml $(DATA)/berror/gfs_gsi_global.nml processor layout x direction: &layout_gsib_x 3 processor layout y direction: &layout_gsib_y 2 debugging mode: false @@ -35,17 +35,17 @@ components: covariance model: ensemble members from template: template: - datetime: '{{BKG_ISOTIME}}' + datetime: '{{ current_cycle | to_isotime }}' filetype: fms restart state variables: *3dvars_anal datapath: ens/mem%mem% - filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res' + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' pattern: '%mem%' - nmembers: !ENV ${NMEM_ENKF} + nmembers: $(NMEM_ENKF) zero padding: 3 # localization: # localization method: SABER diff --git a/parm/atm/berror/staticb_bump.yaml b/parm/atm/berror/staticb_bump.yaml index 27c207d1d..8cabf7060 100644 --- a/parm/atm/berror/staticb_bump.yaml +++ b/parm/atm/berror/staticb_bump.yaml @@ -8,7 +8,7 @@ saber blocks: active variables: &active_vars [psi,chi,t,ps,rh,liq_wat,o3mr] bump: io: - data directory: !ENV &staticb_dir ${DATA}/berror + data directory: &staticb_dir $(DATA)/berror drivers: multivariate strategy: specific_univariate read local nicas: true diff --git a/parm/atm/berror/staticb_gsibec.yaml b/parm/atm/berror/staticb_gsibec.yaml index 66b98b535..ab82efe43 100644 --- a/parm/atm/berror/staticb_gsibec.yaml +++ b/parm/atm/berror/staticb_gsibec.yaml @@ -6,10 +6,10 @@ saber central block: specific_humidity,cloud_liquid_ice,cloud_liquid_water, mole_fraction_of_ozone_in_air] output variables: *control_vars - gsi akbk: !ENV &akbk ${DATA}/fv3jedi/akbk.nc4 - gsi error covariance file: !ENV &gsiberr ${DATA}/berror/gsi-coeffs-gfs-global.nc4 -# gsi error covariance file: !ENV &gsiberr ${DATA}/berror/global_berror.f77 - gsi berror namelist file: !ENV &gsibnml ${DATA}/berror/gfs_gsi_global.nml + gsi akbk: &akbk $(DATA)/fv3jedi/akbk.nc4 + gsi error covariance file: &gsiberr $(DATA)/berror/gsi-coeffs-gfs-global.nc4 +# gsi error covariance file: &gsiberr $(DATA)/berror/global_berror.f77 + gsi berror namelist file: &gsibnml $(DATA)/berror/gfs_gsi_global.nml processor layout x direction: &layout_gsib_x 3 processor layout y direction: &layout_gsib_y 2 debugging mode: false diff --git a/parm/atm/hofx/hofx4d.yaml b/parm/atm/hofx/hofx4d.yaml index 0bd231398..14b6c0e66 100644 --- a/parm/atm/hofx/hofx4d.yaml +++ b/parm/atm/hofx/hofx4d.yaml @@ -1,35 +1,35 @@ -window begin: '{{ATM_WINDOW_BEGIN}}' +window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' window length: $(ATM_WINDOW_LENGTH) forecast length: $(ATM_WINDOW_LENGTH) geometry: fms initialization: - namelist filename: !ENV ${DATA}/fv3jedi/fmsmpp.nml - field table filename: !ENV ${DATA}/fv3jedi/field_table - akbk: !ENV ${DATA}/fv3jedi/akbk.nc4 + namelist filename: $(DATA)/fv3jedi/fmsmpp.nml + field table filename: $(DATA)/fv3jedi/field_table + akbk: $(DATA)/fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: !ENV ${DATA}/fv3jedi/gfs-restart.yaml + field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml initial condition: - datapath: !ENV ${DATA}/bkg + datapath: $(DATA)/bkg filetype: fms restart - datetime: '{{ATM_WINDOW_BEGIN}}' - filename_core: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.coupler.res' + datetime: '{{ ATM_WINDOW_BEGIN | to_isotime }}' + filename_core: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.coupler.res' state variables: [ua,va,t,delp,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] model: name: PSEUDO - datapath: !ENV ${DATA}/bkg + datapath: $(DATA)/bkg filetype: fms restart - datetime: '{{ATM_WINDOW_BEGIN}}' + datetime: '{{ ATM_WINDOW_BEGIN | to_isotime }}' filename is datetime templated: true filename_core: '%yyyy%mm%dd.%hh%MM%ss.fv_core.res.nc' filename_trcr: '%yyyy%mm%dd.%hh%MM%ss.fv_tracer.res.nc' diff --git a/parm/atm/hofx/hofx_nomodel.yaml b/parm/atm/hofx/hofx_nomodel.yaml index b50d23417..f980d53be 100644 --- a/parm/atm/hofx/hofx_nomodel.yaml +++ b/parm/atm/hofx/hofx_nomodel.yaml @@ -1,26 +1,26 @@ -window begin: '{{ATM_WINDOW_BEGIN}}' +window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' window length: $(ATM_WINDOW_LENGTH) geometry: fms initialization: - namelist filename: !ENV ${DATA}/fv3jedi/fmsmpp.nml - field table filename: !ENV ${DATA}/fv3jedi/field_table - akbk: !ENV ${DATA}/fv3jedi/akbk.nc4 + namelist filename: $(DATA)/fv3jedi/fmsmpp.nml + field table filename: $(DATA)/fv3jedi/field_table + akbk: $(DATA)/fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: !ENV ${DATA}/fv3jedi/gfs-restart.yaml + field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml state: - datapath: !ENV ${DATA}/bkg + datapath: $(DATA)/bkg filetype: fms restart - datetime: '{{BKG_ISOTIME}}' - filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res' + datetime: '{{ current_cycle | to_isotime }}' + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' state variables: [ua,va,t,delp,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] diff --git a/parm/atm/obs/config/aircraft.yaml b/parm/atm/obs/config/aircraft.yaml index 460af86b9..4920a90db 100644 --- a/parm/atm/obs/config/aircraft.yaml +++ b/parm/atm/obs/config/aircraft.yaml @@ -3,7 +3,7 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}aircraft.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)aircraft.${{ current_cycle | to_YMDH }}.nc4 obsgrouping: group variables: ["stationIdentification"] sort variable: "pressure" @@ -11,7 +11,7 @@ obs space: obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_aircraft_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_aircraft_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [windEastward, windNorthward, airTemperature, specificHumidity] diff --git a/parm/atm/obs/config/atms_n20.yaml b/parm/atm/obs/config/atms_n20.yaml index 0d9e863d5..98c6e5ffd 100644 --- a/parm/atm/obs/config/atms_n20.yaml +++ b/parm/atm/obs/config/atms_n20.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}atms_n20.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)atms_n20.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_atms_n20_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_atms_n20_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightness_temperature] @@ -20,16 +20,16 @@ obs operator: obs options: Sensor_ID: atms_n20 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}atms_n20.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}atms_n20.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias.${{ previous_cycle | to_YMDH }}.nc4 + output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias.${{ current_cycle | to_YMDH }}.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &atms_n20_tlapse !ENV ${DATA}/obs/${GPREFIX}atms_n20.tlapse.${GDATE}.txt + tlapse: &atms_n20_tlapse $(DATA)/obs/$(GPREFIX)atms_n20.tlapse.${{ previous_cycle | to_YMDH }}.txt - name: lapse_rate tlapse: *atms_n20_tlapse - name: emissivity @@ -46,11 +46,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}atms_n20.satbias_cov.${GDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias_cov.${{ previous_cycle | to_YMDH }}.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}atms_n20.satbias_cov.${CDATE}.nc4 + output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias_cov.${{ current_cycle | to_YMDH }}.nc4 obs filters: - filter: BlackList filter variables: diff --git a/parm/atm/obs/config/cris-fsr_n20.yaml b/parm/atm/obs/config/cris-fsr_n20.yaml index 3d2a43313..22cf9c4d0 100644 --- a/parm/atm/obs/config/cris-fsr_n20.yaml +++ b/parm/atm/obs/config/cris-fsr_n20.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}cris-fsr_n20.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)cris-fsr_n20.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_cris-fsr_n20_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_cris-fsr_n20_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightness_temperature] @@ -51,16 +51,16 @@ obs operator: obs options: Sensor_ID: cris-fsr_n20 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}cris-fsr_n20.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}cris-fsr_n20.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_n20.satbias.${{ previous_cycle | to_YMDH }}.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_n20.satbias.${{ current_cycle | to_YMDH }}.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &cris-fsr_n20_tlapse !ENV ${DATA}/obs/${GPREFIX}cris-fsr_n20.tlapse.${GDATE}.txt + tlapse: &cris-fsr_n20_tlapse $(DATA)/obs/$(GPREFIX)cris-fsr_n20.tlapse.${{ previous_cycle | to_YMDH }}.txt - name: lapse_rate tlapse: *cris-fsr_n20_tlapse - name: emissivity @@ -77,11 +77,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}cris-fsr_n20.satbias_cov.${GDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_n20.satbias_cov.${{ previous_cycle | to_YMDH }}.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}cris-fsr_n20.satbias_cov.${CDATE}.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_n20.satbias_cov.${{ current_cycle | to_YMDH }}.nc4 # #obs filters: ## Wavenumber Check diff --git a/parm/atm/obs/config/cris-fsr_npp.yaml b/parm/atm/obs/config/cris-fsr_npp.yaml index a3943264d..8bfe57576 100644 --- a/parm/atm/obs/config/cris-fsr_npp.yaml +++ b/parm/atm/obs/config/cris-fsr_npp.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}cris-fsr_npp.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)cris-fsr_npp.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_cris-fsr_npp_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_cris-fsr_npp_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightness_temperature] @@ -51,16 +51,16 @@ obs operator: obs options: Sensor_ID: cris-fsr_npp EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}cris-fsr_npp.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}cris-fsr_npp.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_npp.satbias.${{ previous_cycle | to_YMDH }}.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_npp.satbias.${{ current_cycle | to_YMDH }}.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &cris-fsr_npp_tlapse !ENV ${DATA}/obs/${GPREFIX}cris-fsr_npp.tlapse.${GDATE}.txt + tlapse: &cris-fsr_npp_tlapse $(DATA)/obs/$(GPREFIX)cris-fsr_npp.tlapse.${{ previous_cycle | to_YMDH }}.txt - name: lapse_rate tlapse: *cris-fsr_npp_tlapse - name: emissivity @@ -77,11 +77,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}cris-fsr_npp.satbias_cov.${GDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_npp.satbias_cov.${{ previous_cycle | to_YMDH }}.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}cris-fsr_npp.satbias_cov.${CDATE}.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_npp.satbias_cov.${{ current_cycle | to_YMDH }}.nc4 #obs filters: ## Wavenumber Check #- filter: BlackList diff --git a/parm/atm/obs/config/omi_aura.yaml b/parm/atm/obs/config/omi_aura.yaml index e5346e322..79973780c 100644 --- a/parm/atm/obs/config/omi_aura.yaml +++ b/parm/atm/obs/config/omi_aura.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}omi_aura.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)omi_aura.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_omi_aura_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_omi_aura_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [ozoneTotal] diff --git a/parm/atm/obs/config/ompsnp_npp.yaml b/parm/atm/obs/config/ompsnp_npp.yaml index cae446044..5ba772fb3 100644 --- a/parm/atm/obs/config/ompsnp_npp.yaml +++ b/parm/atm/obs/config/ompsnp_npp.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}ompsnp_npp.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ompsnp_npp.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_ompsnp_npp_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_ompsnp_npp_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [ozoneTotal] diff --git a/parm/atm/obs/config/ompstc8_npp.yaml b/parm/atm/obs/config/ompstc8_npp.yaml index c3aee74f4..e3b11ff64 100644 --- a/parm/atm/obs/config/ompstc8_npp.yaml +++ b/parm/atm/obs/config/ompstc8_npp.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}ompstc8_npp.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ompstc8_npp.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_ompstc8_npp_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_ompstc8_npp_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [ozoneTotal] diff --git a/parm/atm/obs/config/satwind.yaml b/parm/atm/obs/config/satwind.yaml index ebef654b4..853cbce77 100644 --- a/parm/atm/obs/config/satwind.yaml +++ b/parm/atm/obs/config/satwind.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}satwind.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)satwind.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_satwind_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_satwind_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] diff --git a/parm/atm/obs/config/sfc.yaml b/parm/atm/obs/config/sfc.yaml index 32a3f1acb..31fd74c0c 100644 --- a/parm/atm/obs/config/sfc.yaml +++ b/parm/atm/obs/config/sfc.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}sfc.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)sfc.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_sfc_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_sfc_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [stationPressure] diff --git a/parm/atm/obs/config/sfcship.yaml b/parm/atm/obs/config/sfcship.yaml index c5bf1b304..c3530f404 100644 --- a/parm/atm/obs/config/sfcship.yaml +++ b/parm/atm/obs/config/sfcship.yaml @@ -3,12 +3,12 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}sfcship.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)sfcship.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File overwrite: true - obsfile: !ENV ${DATA}/diags/diag_sfcship_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_sfcship_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [stationPressure, airTemperature, specificHumidity] diff --git a/parm/atm/obs/config/ssmis_f17.yaml b/parm/atm/obs/config/ssmis_f17.yaml index 12d96ee47..10567fa4b 100644 --- a/parm/atm/obs/config/ssmis_f17.yaml +++ b/parm/atm/obs/config/ssmis_f17.yaml @@ -4,22 +4,22 @@ obs operator: obs options: Sensor_ID: ssmis_f17 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs space: name: ssmis_f17 obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}ssmis_f17.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ssmis_f17.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_ssmis_f17_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_ssmis_f17_${{ current_cycle | to_YMDH }}.nc4 simulated variables: [brightness_temperature] channels: 1-24 obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}ssmis_f17.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}ssmis_f17.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)ssmis_f17.satbias.${{ previous_cycle | to_YMDH }}.nc4 + output file: $(DATA)/bc/$(APREFIX)ssmis_f17.satbias.${{ current_cycle | to_YMDH }}.nc4 variational bc: predictors: - name: constant @@ -36,7 +36,7 @@ obs bias: - name: sine_of_latitude - name: lapse_rate order: 2 - tlapse: &ssmis_f17_tlapse !ENV ${DATA}/obs/${GPREFIX}ssmis_f17.tlapse.${GDATE}.txt + tlapse: &ssmis_f17_tlapse $(DATA)/obs/$(GPREFIX)ssmis_f17.tlapse.${{ previous_cycle | to_YMDH }}.txt - name: lapse_rate tlapse: *ssmis_f17_tlapse - name: emissivity diff --git a/parm/atm/obs/config/ssmis_f18.yaml b/parm/atm/obs/config/ssmis_f18.yaml index 41bdf450a..21e329c75 100644 --- a/parm/atm/obs/config/ssmis_f18.yaml +++ b/parm/atm/obs/config/ssmis_f18.yaml @@ -4,22 +4,22 @@ obs operator: obs options: Sensor_ID: ssmis_f18 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs space: name: ssmis_f18 obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}ssmis_f18.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ssmis_f18.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_ssmis_f18_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_ssmis_f18_${{ current_cycle | to_YMDH }}.nc4 simulated variables: [brightness_temperature] channels: 1-24 obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}ssmis_f18.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}ssmis_f18.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)ssmis_f18.satbias.${{ previous_cycle | to_YMDH }}.nc4 + output file: $(DATA)/bc/$(APREFIX)ssmis_f18.satbias.${{ current_cycle | to_YMDH }}.nc4 variational bc: predictors: - name: constant @@ -36,7 +36,7 @@ obs bias: - name: sine_of_latitude - name: lapse_rate order: 2 - tlapse: &ssmis_f18_tlapse !ENV ${DATA}/obs/${GPREFIX}ssmis_f18.tlapse.${GDATE}.txt + tlapse: &ssmis_f18_tlapse $(DATA)/obs/$(GPREFIX)ssmis_f18.tlapse.${{ previous_cycle | to_YMDH }}.txt - name: lapse_rate tlapse: *ssmis_f18_tlapse - name: emissivity diff --git a/test/soca/CMakeLists.txt b/test/soca/CMakeLists.txt index c9e3efe66..cbbdd3bb6 100644 --- a/test/soca/CMakeLists.txt +++ b/test/soca/CMakeLists.txt @@ -87,7 +87,7 @@ add_test(NAME test_gdasapp_soca_ana_prep WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/3dvar) set_tests_properties(test_gdasapp_soca_ana_prep PROPERTIES - ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH}") + ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_SOURCE_DIR}/../../ush/python/pygw/src:$ENV{PYTHONPATH}") # Test exgdas scripts from the global-worflow if (WORKFLOW_TESTS) diff --git a/ush/run_jedi_exe.py b/ush/run_jedi_exe.py index 6bebd4d01..4c4693f6e 100755 --- a/ush/run_jedi_exe.py +++ b/ush/run_jedi_exe.py @@ -7,6 +7,8 @@ import subprocess import sys import yaml +import pygw +from pygw.yaml_file import parse_j2yaml, save_as_yaml def export_envar(yamlfile, bashout): @@ -77,6 +79,8 @@ def run_jedi_exe(yamlconfig): gprefix = executable_subconfig['dump'] + ".t" + str(gcyc) + "z." comin = executable_subconfig.get('gdas_fix_root', './') comin_ges_ens = os.path.join(comin, 'cases', 'enkfgdas.' + str(gPDY), str(gcyc)) + dump = executable_subconfig['dump'] + output_file = os.path.join(workdir, f"gdas_{app_mode}.yaml") single_exec = True var_config = { @@ -112,6 +116,8 @@ def run_jedi_exe(yamlconfig): 'valid_time': f"{valid_time.strftime('%Y-%m-%dT%H:%M:%SZ')}", 'window_begin': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}", 'prev_valid_time': f"{prev_cycle.strftime('%Y-%m-%dT%H:%M:%SZ')}", + 'previous_cycle': f"{prev_cycle}", + 'current_cycle': f"{valid_time}", 'atm_window_length': executable_subconfig['atm_window_length'], 'ATM_WINDOW_LENGTH': f"PT{assim_freq}H", 'ATM_WINDOW_BEGIN': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}", @@ -127,24 +133,52 @@ def run_jedi_exe(yamlconfig): executable_subconfig['atm_window_length']), 'BKG_TSTEP': executable_subconfig.get('forecast_step', 'PT6H'), 'INTERP_METHOD': executable_subconfig.get('interp_method', 'barycentric'), + 'output_file': os.path.join(workdir, f"gdas_{app_mode}.yaml"), + 'dump': f"{dump}", + 'output_file': f"{output_file}", } - output_file = os.path.join(workdir, f"gdas_{app_mode}.yaml") + # set some environment variables os.environ['PARMgfs'] = os.path.join(all_config_dict['GDASApp home'], 'parm') for key, value in var_config.items(): os.environ[key] = str(value) # generate YAML for executable based on input config logging.info(f'Using yamlconfig {yamlconfig}') - genYAML(yamlconfig, output=output_file) + + local_dict = { + 'npx_ges': f"{int(os.environ['CASE'][1:]) + 1}", + 'npy_ges': f"{int(os.environ['CASE'][1:]) + 1}", + 'npz_ges': f"{int(os.environ['LEVS']) - 1}", + 'npz': f"{int(os.environ['LEVS']) - 1}", + 'npx_anl': f"{int(os.environ['CASE_ANL'][1:]) + 1}", + 'npy_anl': f"{int(os.environ['CASE_ANL'][1:]) + 1}", + 'npz_anl': f"{int(os.environ['LEVS']) - 1}", + 'NMEM_ENKF': f"{int(os.environ['NMEM_ENKF'])}", + 'ATM_WINDOW_BEGIN': window_begin, + 'ATM_WINDOW_LENGTH': f"PT{assim_freq}H", + 'BKG_TSTEP': executable_subconfig.get('forecast_step', 'PT6H'), + 'OPREFIX': f"{dump}.t{cyc}z.", # TODO: CDUMP is being replaced by RUN + 'APREFIX': f"{dump}.t{cyc}z.", # TODO: CDUMP is being replaced by RUN + 'GPREFIX': f"gdas.t{gcyc}z.", + 'DATA': os.path.join(workdir), + 'layout_x': str(executable_subconfig['layout_x']), + 'layout_y': str(executable_subconfig['layout_y']), + 'previous_cycle': prev_cycle, + 'current_cycle': valid_time, + } + + varda_yaml = parse_j2yaml(all_config_dict['template'], local_dict) + save_as_yaml(varda_yaml, output_file) + logging.info(f'Wrote YAML file to {output_file}') # use R2D2 to stage backgrounds, obs, bias correction files, etc. if app_mode in ['variational', 'hofx']: - ufsda.stage.gdas_single_cycle(var_config) + ufsda.stage.gdas_single_cycle(var_config, local_dict) # stage ensemble backgrouns for letkf if app_mode in ['letkf']: ufsda.stage.background_ens(var_config) - ufsda.stage.atm_obs(var_config) - ufsda.stage.bias_obs(var_config) + ufsda.stage.atm_obs(var_config, local_dict) + ufsda.stage.bias_obs(var_config, local_dict) # link additional fix files needed (CRTM, fieldmetadata, etc.) gdasfix = executable_subconfig['gdas_fix_root'] diff --git a/ush/ufsda/stage.py b/ush/ufsda/stage.py index b6863ab5f..9606cd740 100644 --- a/ush/ufsda/stage.py +++ b/ush/ufsda/stage.py @@ -12,7 +12,8 @@ import logging import glob import numpy as np -from pygw.yaml_file import YAMLFile +import yaml +from pygw.yaml_file import YAMLFile, parse_j2yaml import ufsda.soca_utils __all__ = ['atm_background', 'atm_obs', 'bias_obs', 'background', 'background_ens', 'fv3jedi', 'obs', 'berror', 'gdas_fix', 'gdas_single_cycle'] @@ -146,7 +147,7 @@ def atm_background(config): ufsda.r2d2.fetch(r2d2_config) -def atm_obs(config): +def atm_obs(config, local_dict): # fetch atm analysis obs r2d2_config = { 'start': config['prev_valid_time'], @@ -159,22 +160,25 @@ def atm_obs(config): r2d2_config = NiceDict(r2d2_config) # get list of obs to process and their output files obs_list_yaml = config['OBS_LIST'] - obs_list_config = YAMLFile(path=obs_list_yaml) + obs_list_config = Configuration(obs_list_yaml) + obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config) + for ob in obs_list_config['observers']: + ob_config = parse_j2yaml(ob, local_dict) # first get obs r2d2_config.pop('file_type', None) r2d2_config['type'] = 'ob' r2d2_config['provider'] = 'ncdiag' r2d2_config['start'] = config['ATM_WINDOW_BEGIN'] r2d2_config['end'] = r2d2_config['start'] - ob_basename = os.path.basename(ob['obs space']['obsdatain']['engine']['obsfile']) + ob_basename = os.path.basename(ob_config['obs space']['obsdatain']['engine']['obsfile']) target_file = os.path.join(os.environ['COMOUT'], ob_basename) r2d2_config['target_file_fmt'] = target_file - r2d2_config['obs_types'] = [ob['obs space']['name']] + r2d2_config['obs_types'] = [ob_config['obs space']['name']] ufsda.r2d2.fetch(r2d2_config) -def bias_obs(config): +def bias_obs(config, local_dict): # fetch bias files r2d2_config = { 'start': config['prev_valid_time'], @@ -187,12 +191,15 @@ def bias_obs(config): r2d2_config = NiceDict(r2d2_config) # get list of obs to process and their output files obs_list_yaml = config['OBS_LIST'] - obs_list_config = YAMLFile(path=obs_list_yaml) + obs_list_config = Configuration(obs_list_yaml) + obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config) + for ob in obs_list_config['observers']: + ob_config = parse_j2yaml(ob, local_dict) r2d2_config.pop('file_type', None) - r2d2_config['obs_types'] = [ob['obs space']['name']] + r2d2_config['obs_types'] = [ob_config['obs space']['name']] # get bias files if needed - if 'obs bias' in ob.keys(): + if 'obs bias' in ob_config.keys(): r2d2_config['type'] = 'bc' r2d2_config['provider'] = 'gsi' r2d2_config['start'] = config['prev_valid_time'] @@ -200,7 +207,7 @@ def bias_obs(config): # fetch satbias r2d2_config['file_type'] = 'satbias' - target_file = ob['obs bias']['input file'] + target_file = ob_config['obs bias']['input file'] ob_basename = os.path.basename(target_file) target_file = os.path.join(os.environ['COMOUT'], ob_basename) r2d2_config['target_file_fmt'] = target_file @@ -230,7 +237,7 @@ def bias_obs(config): ufsda.r2d2.fetch(r2d2_config) -def gdas_single_cycle(config): +def gdas_single_cycle(config, local_dict): # grab backgrounds first r2d2_config = { 'start': config['prev_valid_time'], @@ -267,8 +274,9 @@ def gdas_single_cycle(config): obs_list_yaml = config['OBS_LIST'] obs_list_config = Configuration(obs_list_yaml) obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config) + for ob in obs_list_config['observers']: - ob_config = YAMLFile(path=ob) + ob_config = parse_j2yaml(ob, local_dict) # first get obs r2d2_config.pop('file_type', None) r2d2_config['type'] = 'ob' From df45b6712c1c0ef9be76443b616e9719deb6426e Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 12 Apr 2023 14:47:31 +0000 Subject: [PATCH 09/13] remove 10-digit datetime string from radiance bias correction filenames (#357) --- parm/atm/obs/config/amsua_n19.yaml | 10 +++++----- parm/atm/obs/config/atms_n20.yaml | 10 +++++----- parm/atm/obs/config/cris-fsr_n20.yaml | 10 +++++----- parm/atm/obs/config/cris-fsr_npp.yaml | 10 +++++----- parm/atm/obs/config/lgetkf_amsua_n19.yaml | 10 +++++----- parm/atm/obs/config/ssmis_f17.yaml | 6 +++--- parm/atm/obs/config/ssmis_f18.yaml | 6 +++--- 7 files changed, 31 insertions(+), 31 deletions(-) diff --git a/parm/atm/obs/config/amsua_n19.yaml b/parm/atm/obs/config/amsua_n19.yaml index f22eecdb8..df6c38528 100644 --- a/parm/atm/obs/config/amsua_n19.yaml +++ b/parm/atm/obs/config/amsua_n19.yaml @@ -22,14 +22,14 @@ obs operator: EndianType: little_endian CoefficientPath: $(DATA)/crtm/ obs bias: - input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias.{{ previous_cycle | to_YMDH }}.nc4 - output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias.{{ current_cycle | to_YMDH }}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &amsua_n19_tlapse $(DATA)/obs/$(GPREFIX)amsua_n19.tlapse.{{ previous_cycle | to_YMDH }}.txt + tlapse: &amsua_n19_tlapse $(DATA)/obs/$(GPREFIX)amsua_n19.tlapse.txt - name: lapse_rate tlapse: *amsua_n19_tlapse - name: emissivity @@ -46,11 +46,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias_cov.{{ previous_cycle | to_YMDH }}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias_cov.{{ current_cycle | to_YMDH }}.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias_cov.nc4 obs filters: - filter: BlackList filter variables: diff --git a/parm/atm/obs/config/atms_n20.yaml b/parm/atm/obs/config/atms_n20.yaml index 98c6e5ffd..28b19e692 100644 --- a/parm/atm/obs/config/atms_n20.yaml +++ b/parm/atm/obs/config/atms_n20.yaml @@ -22,14 +22,14 @@ obs operator: EndianType: little_endian CoefficientPath: $(DATA)/crtm/ obs bias: - input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias.${{ previous_cycle | to_YMDH }}.nc4 - output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias.${{ current_cycle | to_YMDH }}.nc4 + input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &atms_n20_tlapse $(DATA)/obs/$(GPREFIX)atms_n20.tlapse.${{ previous_cycle | to_YMDH }}.txt + tlapse: &atms_n20_tlapse $(DATA)/obs/$(GPREFIX)atms_n20.tlapse.txt - name: lapse_rate tlapse: *atms_n20_tlapse - name: emissivity @@ -46,11 +46,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias_cov.${{ previous_cycle | to_YMDH }}.nc4 + input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias_cov.${{ current_cycle | to_YMDH }}.nc4 + output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias_cov.nc4 obs filters: - filter: BlackList filter variables: diff --git a/parm/atm/obs/config/cris-fsr_n20.yaml b/parm/atm/obs/config/cris-fsr_n20.yaml index 22cf9c4d0..579439862 100644 --- a/parm/atm/obs/config/cris-fsr_n20.yaml +++ b/parm/atm/obs/config/cris-fsr_n20.yaml @@ -53,14 +53,14 @@ obs operator: EndianType: little_endian CoefficientPath: $(DATA)/crtm/ obs bias: - input file: $(DATA)/obs/$(GPREFIX)cris-fsr_n20.satbias.${{ previous_cycle | to_YMDH }}.nc4 - output file: $(DATA)/bc/$(APREFIX)cris-fsr_n20.satbias.${{ current_cycle | to_YMDH }}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_n20.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_n20.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &cris-fsr_n20_tlapse $(DATA)/obs/$(GPREFIX)cris-fsr_n20.tlapse.${{ previous_cycle | to_YMDH }}.txt + tlapse: &cris-fsr_n20_tlapse $(DATA)/obs/$(GPREFIX)cris-fsr_n20.tlapse.txt - name: lapse_rate tlapse: *cris-fsr_n20_tlapse - name: emissivity @@ -77,11 +77,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: $(DATA)/obs/$(GPREFIX)cris-fsr_n20.satbias_cov.${{ previous_cycle | to_YMDH }}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_n20.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: $(DATA)/bc/$(APREFIX)cris-fsr_n20.satbias_cov.${{ current_cycle | to_YMDH }}.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_n20.satbias_cov.nc4 # #obs filters: ## Wavenumber Check diff --git a/parm/atm/obs/config/cris-fsr_npp.yaml b/parm/atm/obs/config/cris-fsr_npp.yaml index 8bfe57576..55409f454 100644 --- a/parm/atm/obs/config/cris-fsr_npp.yaml +++ b/parm/atm/obs/config/cris-fsr_npp.yaml @@ -53,14 +53,14 @@ obs operator: EndianType: little_endian CoefficientPath: $(DATA)/crtm/ obs bias: - input file: $(DATA)/obs/$(GPREFIX)cris-fsr_npp.satbias.${{ previous_cycle | to_YMDH }}.nc4 - output file: $(DATA)/bc/$(APREFIX)cris-fsr_npp.satbias.${{ current_cycle | to_YMDH }}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_npp.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_npp.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &cris-fsr_npp_tlapse $(DATA)/obs/$(GPREFIX)cris-fsr_npp.tlapse.${{ previous_cycle | to_YMDH }}.txt + tlapse: &cris-fsr_npp_tlapse $(DATA)/obs/$(GPREFIX)cris-fsr_npp.tlapse.txt - name: lapse_rate tlapse: *cris-fsr_npp_tlapse - name: emissivity @@ -77,11 +77,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: $(DATA)/obs/$(GPREFIX)cris-fsr_npp.satbias_cov.${{ previous_cycle | to_YMDH }}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_npp.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: $(DATA)/bc/$(APREFIX)cris-fsr_npp.satbias_cov.${{ current_cycle | to_YMDH }}.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_npp.satbias_cov.nc4 #obs filters: ## Wavenumber Check #- filter: BlackList diff --git a/parm/atm/obs/config/lgetkf_amsua_n19.yaml b/parm/atm/obs/config/lgetkf_amsua_n19.yaml index 78fd790f7..d5fe9691a 100644 --- a/parm/atm/obs/config/lgetkf_amsua_n19.yaml +++ b/parm/atm/obs/config/lgetkf_amsua_n19.yaml @@ -27,14 +27,14 @@ obs operator: obs error: covariance model: diagonal obs bias: - input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias.{{ previous_cycle | to_YMDH }}.nc4 - output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias.{{ current_cycle | to_YMDH }}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &amsua_n19_tlapse $(DATA)/obs/$(GPREFIX)amsua_n19.tlapse.{{ previous_cycle | to_YMDH }}.txt + tlapse: &amsua_n19_tlapse $(DATA)/obs/$(GPREFIX)amsua_n19.tlapse.txt - name: lapse_rate tlapse: *amsua_n19_tlapse - name: emissivity @@ -51,11 +51,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias_cov.{{ previous_cycle | to_YMDH }}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias_cov.{{ current_cycle | to_YMDH }}.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias_cov.nc4 obs filters: - filter: Bounds Check filter variables: diff --git a/parm/atm/obs/config/ssmis_f17.yaml b/parm/atm/obs/config/ssmis_f17.yaml index 10567fa4b..5054951ef 100644 --- a/parm/atm/obs/config/ssmis_f17.yaml +++ b/parm/atm/obs/config/ssmis_f17.yaml @@ -18,8 +18,8 @@ obs space: simulated variables: [brightness_temperature] channels: 1-24 obs bias: - input file: $(DATA)/obs/$(GPREFIX)ssmis_f17.satbias.${{ previous_cycle | to_YMDH }}.nc4 - output file: $(DATA)/bc/$(APREFIX)ssmis_f17.satbias.${{ current_cycle | to_YMDH }}.nc4 + input file: $(DATA)/obs/$(GPREFIX)ssmis_f17.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)ssmis_f17.satbias.nc4 variational bc: predictors: - name: constant @@ -36,7 +36,7 @@ obs bias: - name: sine_of_latitude - name: lapse_rate order: 2 - tlapse: &ssmis_f17_tlapse $(DATA)/obs/$(GPREFIX)ssmis_f17.tlapse.${{ previous_cycle | to_YMDH }}.txt + tlapse: &ssmis_f17_tlapse $(DATA)/obs/$(GPREFIX)ssmis_f17.tlapse.txt - name: lapse_rate tlapse: *ssmis_f17_tlapse - name: emissivity diff --git a/parm/atm/obs/config/ssmis_f18.yaml b/parm/atm/obs/config/ssmis_f18.yaml index 21e329c75..2c757268a 100644 --- a/parm/atm/obs/config/ssmis_f18.yaml +++ b/parm/atm/obs/config/ssmis_f18.yaml @@ -18,8 +18,8 @@ obs space: simulated variables: [brightness_temperature] channels: 1-24 obs bias: - input file: $(DATA)/obs/$(GPREFIX)ssmis_f18.satbias.${{ previous_cycle | to_YMDH }}.nc4 - output file: $(DATA)/bc/$(APREFIX)ssmis_f18.satbias.${{ current_cycle | to_YMDH }}.nc4 + input file: $(DATA)/obs/$(GPREFIX)ssmis_f18.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)ssmis_f18.satbias.nc4 variational bc: predictors: - name: constant @@ -36,7 +36,7 @@ obs bias: - name: sine_of_latitude - name: lapse_rate order: 2 - tlapse: &ssmis_f18_tlapse $(DATA)/obs/$(GPREFIX)ssmis_f18.tlapse.${{ previous_cycle | to_YMDH }}.txt + tlapse: &ssmis_f18_tlapse $(DATA)/obs/$(GPREFIX)ssmis_f18.tlapse.txt - name: lapse_rate tlapse: *ssmis_f18_tlapse - name: emissivity From bd219819d079d56fc834eaa8083d5681469ca086 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 20 Apr 2023 15:45:09 +0000 Subject: [PATCH 10/13] replace JEDIVAREXE with JEDIEXE in config.atmanl (#357) --- test/atm/global-workflow/config.atmanl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/atm/global-workflow/config.atmanl b/test/atm/global-workflow/config.atmanl index edb473045..d7039cd8d 100755 --- a/test/atm/global-workflow/config.atmanl +++ b/test/atm/global-workflow/config.atmanl @@ -25,7 +25,7 @@ export layout_y=1 export io_layout_x=1 export io_layout_y=1 -export JEDIVAREXE=${HOMEgfs}/exec/fv3jedi_var.x +export JEDIEXE=${HOMEgfs}/exec/fv3jedi_var.x export crtm_VERSION="2.3.0" echo "END: config.atmanl" From dc5db4a0106930f835008f69dfc38db36b2fa7cf Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 21 Apr 2023 19:19:13 +0000 Subject: [PATCH 11/13] remove extraneous yaml import (#357) --- ush/ufsda/stage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/ufsda/stage.py b/ush/ufsda/stage.py index 9606cd740..0be2e9562 100644 --- a/ush/ufsda/stage.py +++ b/ush/ufsda/stage.py @@ -12,7 +12,6 @@ import logging import glob import numpy as np -import yaml from pygw.yaml_file import YAMLFile, parse_j2yaml import ufsda.soca_utils From 1e7d993170aa09b3068b85729b092ea015640205 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Sat, 22 Apr 2023 12:09:19 +0000 Subject: [PATCH 12/13] add g-w pygw to PYTHONPATH in run_jedi_exe scripts (#357) --- test/atm/run_jedi_exe_3denvar.sh | 7 +++++++ test/atm/run_jedi_exe_3dhofx.sh | 7 +++++++ test/atm/run_jedi_exe_3dvar.sh | 7 +++++++ test/atm/run_jedi_exe_letkf.sh | 7 +++++++ 4 files changed, 28 insertions(+) diff --git a/test/atm/run_jedi_exe_3denvar.sh b/test/atm/run_jedi_exe_3denvar.sh index 3f716dc42..faa6d09dc 100755 --- a/test/atm/run_jedi_exe_3denvar.sh +++ b/test/atm/run_jedi_exe_3denvar.sh @@ -30,6 +30,13 @@ elif [ "$machine" = "orion" ]; then gdasfix="/work2/noaa/da/cmartin/GDASApp/fix" fi +# Setup python path for workflow utilities and tasks +export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +echo $HOMEgfs +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # Create test run directory mkdir -p ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_3denvar cd ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_3denvar diff --git a/test/atm/run_jedi_exe_3dhofx.sh b/test/atm/run_jedi_exe_3dhofx.sh index 84a9b057a..f55fb6415 100755 --- a/test/atm/run_jedi_exe_3dhofx.sh +++ b/test/atm/run_jedi_exe_3dhofx.sh @@ -30,6 +30,13 @@ elif [ "$machine" = "orion" ]; then gdasfix="/work2/noaa/da/cmartin/GDASApp/fix" fi +# Setup python path for workflow utilities and tasks +export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +echo $HOMEgfs +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # Create test run directory mkdir -p ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_hofx3d cd ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_hofx3d diff --git a/test/atm/run_jedi_exe_3dvar.sh b/test/atm/run_jedi_exe_3dvar.sh index ffc50816e..d333db3cc 100755 --- a/test/atm/run_jedi_exe_3dvar.sh +++ b/test/atm/run_jedi_exe_3dvar.sh @@ -30,6 +30,13 @@ elif [ "$machine" = "orion" ]; then gdasfix="/work2/noaa/da/cmartin/GDASApp/fix" fi +# Setup python path for workflow utilities and tasks +export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +echo $HOMEgfs +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # Create test run directory mkdir -p ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_3dvar cd ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_3dvar diff --git a/test/atm/run_jedi_exe_letkf.sh b/test/atm/run_jedi_exe_letkf.sh index 6039ebcf6..287b67641 100755 --- a/test/atm/run_jedi_exe_letkf.sh +++ b/test/atm/run_jedi_exe_letkf.sh @@ -30,6 +30,13 @@ elif [ "$machine" = "orion" ]; then gdasfix="/work2/noaa/da/cmartin/GDASApp/fix" fi +# Setup python path for workflow utilities and tasks +export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +echo $HOMEgfs +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # Create test run directory mkdir -p ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_letkf cd ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_letkf From 709d0478fad39bbca0a59f4f759940b6a11e3286 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Mon, 24 Apr 2023 15:18:03 +0000 Subject: [PATCH 13/13] move atm run_jedi_exe ctests into global-workflow (#357) --- test/atm/CMakeLists.txt | 20 ---------------- test/atm/global-workflow/CMakeLists.txt | 24 +++++++++++++++++++ .../run_jedi_exe_3denvar.sh | 0 .../run_jedi_exe_3dhofx.sh | 0 .../run_jedi_exe_3dvar.sh | 0 .../run_jedi_exe_letkf.sh | 0 6 files changed, 24 insertions(+), 20 deletions(-) rename test/atm/{ => global-workflow}/run_jedi_exe_3denvar.sh (100%) rename test/atm/{ => global-workflow}/run_jedi_exe_3dhofx.sh (100%) rename test/atm/{ => global-workflow}/run_jedi_exe_3dvar.sh (100%) rename test/atm/{ => global-workflow}/run_jedi_exe_letkf.sh (100%) diff --git a/test/atm/CMakeLists.txt b/test/atm/CMakeLists.txt index e429c321a..117e2ea8e 100644 --- a/test/atm/CMakeLists.txt +++ b/test/atm/CMakeLists.txt @@ -25,26 +25,6 @@ if (BUILD_GDASBUNDLE) ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH}" ) - # test for ush/run_jedi_exe.py 3dhofx - add_test(NAME test_gdasapp_run_jedi_exe_3dhofx - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/run_jedi_exe_3dhofx.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) - - # test for ush/run_jedi_exe.py 3dvar - add_test(NAME test_gdasapp_run_jedi_exe_3dvar - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/run_jedi_exe_3dvar.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) - - # test for ush/run_jedi_exe.py 3denvar - add_test(NAME test_gdasapp_run_jedi_exe_3denvar - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/run_jedi_exe_3denvar.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) - - # test for ush/run_jedi_exe.py letkf - add_test(NAME test_gdasapp_run_jedi_exe_letkf - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/run_jedi_exe_letkf.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) - # Test exgdas scripts from the global-worflow if (WORKFLOW_TESTS) add_subdirectory(global-workflow) diff --git a/test/atm/global-workflow/CMakeLists.txt b/test/atm/global-workflow/CMakeLists.txt index 8a1708a8f..850983f97 100644 --- a/test/atm/global-workflow/CMakeLists.txt +++ b/test/atm/global-workflow/CMakeLists.txt @@ -1,6 +1,30 @@ # test for creating an experiment directory within the global-workflow file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) +# test for ush/run_jedi_exe.py 3dhofx +add_test(NAME test_gdasapp_run_jedi_exe_3dhofx + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_3dhofx.sh + ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) + +# test for ush/run_jedi_exe.py 3dvar +add_test(NAME test_gdasapp_run_jedi_exe_3dvar + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_3dvar.sh + ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) + +# test for ush/run_jedi_exe.py 3denvar +add_test(NAME test_gdasapp_run_jedi_exe_3denvar + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_3denvar.sh + ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) + +# test for ush/run_jedi_exe.py letkf +add_test(NAME test_gdasapp_run_jedi_exe_letkf + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_letkf.sh + ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) + add_test(NAME test_gdasapp_setup_atm_cycled_exp COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/setup_workflow_exp.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} diff --git a/test/atm/run_jedi_exe_3denvar.sh b/test/atm/global-workflow/run_jedi_exe_3denvar.sh similarity index 100% rename from test/atm/run_jedi_exe_3denvar.sh rename to test/atm/global-workflow/run_jedi_exe_3denvar.sh diff --git a/test/atm/run_jedi_exe_3dhofx.sh b/test/atm/global-workflow/run_jedi_exe_3dhofx.sh similarity index 100% rename from test/atm/run_jedi_exe_3dhofx.sh rename to test/atm/global-workflow/run_jedi_exe_3dhofx.sh diff --git a/test/atm/run_jedi_exe_3dvar.sh b/test/atm/global-workflow/run_jedi_exe_3dvar.sh similarity index 100% rename from test/atm/run_jedi_exe_3dvar.sh rename to test/atm/global-workflow/run_jedi_exe_3dvar.sh diff --git a/test/atm/run_jedi_exe_letkf.sh b/test/atm/global-workflow/run_jedi_exe_letkf.sh similarity index 100% rename from test/atm/run_jedi_exe_letkf.sh rename to test/atm/global-workflow/run_jedi_exe_letkf.sh