From 2a189d3f5b2c95d6c15a1e109d03b87ca89bcaa6 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 6 May 2024 16:26:51 +0000 Subject: [PATCH 1/7] Initial commit --- parm/atm/lgetkf/lgetkf.yaml.j2 | 7 ++++--- parm/atm/utils/fv3jedi_fv3inc_lgetkf.yaml.j2 | 17 ++++++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/parm/atm/lgetkf/lgetkf.yaml.j2 b/parm/atm/lgetkf/lgetkf.yaml.j2 index 4f156d213..4368b413b 100644 --- a/parm/atm/lgetkf/lgetkf.yaml.j2 +++ b/parm/atm/lgetkf/lgetkf.yaml.j2 @@ -67,6 +67,7 @@ local ensemble DA: # filename: ./mem%{member}%/atmanl. output ensemble increments: - filetype: auxgrid - gridtype: gaussian - filename: ./anl/mem%{member}%/atminc. + filetype: cube sphere history + filename: ./anl/mem%{member}%/atminc.{{ current_cycle | to_fv3time }}.nc4 + provider: ufs +# fields to write: [ua,va,delp,t,ps,sphum,ice_wat,liq_wat,o3mr] \ No newline at end of file diff --git a/parm/atm/utils/fv3jedi_fv3inc_lgetkf.yaml.j2 b/parm/atm/utils/fv3jedi_fv3inc_lgetkf.yaml.j2 index 786dfd171..8bc5ef2f4 100644 --- a/parm/atm/utils/fv3jedi_fv3inc_lgetkf.yaml.j2 +++ b/parm/atm/utils/fv3jedi_fv3inc_lgetkf.yaml.j2 @@ -1,6 +1,6 @@ variable change: variable change name: Model2GeoVaLs - input variables: &bkgvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr,surface_geopotential_height] + input variables: &bkgvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr,phis] output variables: &fv3incrvars [ua,va,t,sphum,ice_wat,liq_wat,o3mr,delp,hydrostatic_delz] jedi increment variables: [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr] fv3 increment variables: *fv3incrvars @@ -15,7 +15,7 @@ background geometry: npx: {{ npx_ges }} npy: {{ npy_ges }} npz: {{ npz_ges }} - field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_history.yaml + field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml jedi increment geometry: fms initialization: namelist filename: ./fv3jedi/fmsmpp.nml @@ -27,7 +27,7 @@ jedi increment geometry: npx: {{ npx_ges }} npy: {{ npy_ges }} npz: {{ npz_ges }} - field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_history.yaml + field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml fv3 increment geometry: fms initialization: namelist filename: ./fv3jedi/fmsmpp.nml @@ -43,11 +43,14 @@ fv3 increment geometry: members from template: template: background input: - datetime: '{{ current_cycle | to_isotime }}' - filetype: cube sphere history - provider: ufs datapath: ./bkg/mem%mem% - filename: {{ EPREFIX }}atmf006.nc + filetype: fms restart + datetime: '{{ current_cycle | to_isotime }}' + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' state variables: *bkgvars jedi increment input: filetype: cube sphere history From c697fbf3cefe273a54764311f148e21da66c84da Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 6 May 2024 16:30:06 +0000 Subject: [PATCH 2/7] Fix EOL --- parm/atm/lgetkf/lgetkf.yaml.j2 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/parm/atm/lgetkf/lgetkf.yaml.j2 b/parm/atm/lgetkf/lgetkf.yaml.j2 index 4368b413b..2da1a51d9 100644 --- a/parm/atm/lgetkf/lgetkf.yaml.j2 +++ b/parm/atm/lgetkf/lgetkf.yaml.j2 @@ -1,5 +1,5 @@ geometry: - fms initialization: +1;95;0c fms initialization: namelist filename: ./fv3jedi/fmsmpp.nml field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 @@ -69,5 +69,4 @@ local ensemble DA: output ensemble increments: filetype: cube sphere history filename: ./anl/mem%{member}%/atminc.{{ current_cycle | to_fv3time }}.nc4 - provider: ufs -# fields to write: [ua,va,delp,t,ps,sphum,ice_wat,liq_wat,o3mr] \ No newline at end of file + provider: ufs \ No newline at end of file From 6e2731d4fe69089e430c72704aaa62253ea859cc Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Mon, 6 May 2024 12:31:52 -0400 Subject: [PATCH 3/7] Update lgetkf.yaml.j2 --- parm/atm/lgetkf/lgetkf.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/atm/lgetkf/lgetkf.yaml.j2 b/parm/atm/lgetkf/lgetkf.yaml.j2 index 2da1a51d9..1b2d49dbc 100644 --- a/parm/atm/lgetkf/lgetkf.yaml.j2 +++ b/parm/atm/lgetkf/lgetkf.yaml.j2 @@ -1,5 +1,5 @@ geometry: -1;95;0c fms initialization: + fms initialization: namelist filename: ./fv3jedi/fmsmpp.nml field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 @@ -69,4 +69,4 @@ local ensemble DA: output ensemble increments: filetype: cube sphere history filename: ./anl/mem%{member}%/atminc.{{ current_cycle | to_fv3time }}.nc4 - provider: ufs \ No newline at end of file + provider: ufs From 991e8f9e27fd492c6213fe270938f1d67e4ae158 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 6 May 2024 21:12:08 +0000 Subject: [PATCH 4/7] Add new jjob test for fv3inc ens job --- test/atm/global-workflow/CMakeLists.txt | 5 ++ test/atm/global-workflow/jjob_ens_inc.sh | 58 ++++++++++++++++++++++++ test/atm/global-workflow/jjob_ens_run.sh | 6 +-- 3 files changed, 66 insertions(+), 3 deletions(-) create mode 100755 test/atm/global-workflow/jjob_ens_inc.sh diff --git a/test/atm/global-workflow/CMakeLists.txt b/test/atm/global-workflow/CMakeLists.txt index cda1d56d4..1a91b0740 100644 --- a/test/atm/global-workflow/CMakeLists.txt +++ b/test/atm/global-workflow/CMakeLists.txt @@ -34,6 +34,11 @@ add_test(NAME test_gdasapp_atm_jjob_ens_init add_test(NAME test_gdasapp_atm_jjob_ens_run COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_run.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) + +add_test(NAME test_gdasapp_atm_jjob_ens_inc + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_inc.sh + ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) add_test(NAME test_gdasapp_atm_jjob_ens_final diff --git a/test/atm/global-workflow/jjob_ens_inc.sh b/test/atm/global-workflow/jjob_ens_inc.sh new file mode 100755 index 000000000..b0eadfd7a --- /dev/null +++ b/test/atm/global-workflow/jjob_ens_inc.sh @@ -0,0 +1,58 @@ +#! /usr/bin/env bash + +set -x +bindir=$1 +srcdir=$2 + +# Set g-w HOMEgfs +topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) +export HOMEgfs=$topdir + +# Set variables for ctest +export PSLOT=gdas_test +export EXPDIR=$bindir/test/atm/global-workflow/testrun/experiments/$PSLOT +export PDY=20210323 +export cyc=18 +export CDATE=${PDY}${cyc} +#export gPDY=20210323 +#export gcyc=12 +#export GDATE=${gPDY}${gcyc} +export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS/$PSLOT +export RUN=enkfgdas +export CDUMP=enkfgdas +export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIRS/$PSLOT +export COMIN_GES=${bindir}/test/atm/bkg +export pid=${pid:-$$} +export jobid=$pid +export COMROOT=$DATAROOT +export NMEM_ENS=3 +export ACCOUNT=da-cpu + +# Set python path for workflow utilities and tasks +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" +export PYTHONPATH + +# Detemine machine from config.base +machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') + +# Set NETCDF and UTILROOT variables (used in config.base) +if [[ $machine = 'HERA' ]]; then + NETCDF=$( which ncdump ) + export NETCDF + export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then + ncdump=$( which ncdump ) + NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) + export NETCDF + export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2 +fi + +# Execute j-job +if [[ $machine = 'HERA' ]]; then + sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then + sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN +else + ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT +fi diff --git a/test/atm/global-workflow/jjob_ens_run.sh b/test/atm/global-workflow/jjob_ens_run.sh index 6844e6973..b8338d327 100755 --- a/test/atm/global-workflow/jjob_ens_run.sh +++ b/test/atm/global-workflow/jjob_ens_run.sh @@ -50,9 +50,9 @@ fi # Execute j-job if [[ $machine = 'HERA' ]]; then - sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN + sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then - sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN + sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF else - ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN + ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF fi From cd3b9a0b361be7cf75467d3fab0f48a5a84bbdbc Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 7 May 2024 13:52:33 -0500 Subject: [PATCH 5/7] Fixed small bug --- test/atm/global-workflow/jjob_ens_inc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/atm/global-workflow/jjob_ens_inc.sh b/test/atm/global-workflow/jjob_ens_inc.sh index b0eadfd7a..64b320083 100755 --- a/test/atm/global-workflow/jjob_ens_inc.sh +++ b/test/atm/global-workflow/jjob_ens_inc.sh @@ -50,9 +50,9 @@ fi # Execute j-job if [[ $machine = 'HERA' ]]; then - sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN + sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then - sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN + sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT fi From c098d53ac56f4032c83e45621617a57a9e807a84 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 8 May 2024 15:33:25 +0000 Subject: [PATCH 6/7] Clean up --- test/atm/global-workflow/jjob_ens_inc.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/atm/global-workflow/jjob_ens_inc.sh b/test/atm/global-workflow/jjob_ens_inc.sh index 64b320083..c7c885f02 100755 --- a/test/atm/global-workflow/jjob_ens_inc.sh +++ b/test/atm/global-workflow/jjob_ens_inc.sh @@ -14,9 +14,6 @@ export EXPDIR=$bindir/test/atm/global-workflow/testrun/experiments/$PSLOT export PDY=20210323 export cyc=18 export CDATE=${PDY}${cyc} -#export gPDY=20210323 -#export gcyc=12 -#export GDATE=${gPDY}${gcyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS/$PSLOT export RUN=enkfgdas export CDUMP=enkfgdas From 76d5c5ecd5d199dbb837fab0e1dee8193c4ccb09 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 9 May 2024 13:53:12 +0000 Subject: [PATCH 7/7] Update --- parm/atm/jcb-base.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/atm/jcb-base.yaml.j2 b/parm/atm/jcb-base.yaml.j2 index 4845b0fbc..c8b85d564 100644 --- a/parm/atm/jcb-base.yaml.j2 +++ b/parm/atm/jcb-base.yaml.j2 @@ -11,7 +11,7 @@ app_path_observation_chronicle: "{{PARMgfs}}/gdas/jcb-gdas/observation_chronicle # ------------------------------------------------------------ #final_increment_file: final_increment_gaussian final_increment_file: final_increment_cubed_sphere -output_ensemble_increments_file: output_ensemble_increments_gaussian +output_ensemble_increments_file: output_ensemble_increments_cubed_sphere model_file: model_pseudo initial_condition_file: background # Initial conditions for 4D apps is background