Add 3denvar option to run_jedi_exe.py

CMakeLists.txt

@@ -67,7 +67,7 @@ if(BUILD_GDASBUNDLE)
 # Build GSI-B
   option(BUILD_GSIBEC "Build GSI-B" OFF)
   if(BUILD_GSIBEC)
-    ecbuild_bundle( PROJECT gsibec GIT "https://github.com/GEOS-ESM/GSIbec.git" BRANCH develop )
+    ecbuild_bundle( PROJECT gsibec GIT "https://github.com/GEOS-ESM/GSIbec.git" TAG 1.0.7 )
   endif()
 
 # Core JEDI repositories

parm/atm/berror/hybvar_gsibec.yaml

@@ -0,0 +1,67 @@
+covariance model: hybrid
+components:
+- covariance:
+    covariance model: SABER
+    saber central block:
+      saber block name: gsi covariance
+      input variables: &control_vars [eastward_wind,northward_wind,air_temperature,surface_pressure,
+                                      specific_humidity,cloud_liquid_ice,cloud_liquid_water,
+                                      mole_fraction_of_ozone_in_air]
+      output variables: *control_vars
+      gsi akbk: !ENV &akbk ${DATA}/fv3jedi/akbk.nc4
+      gsi error covariance file: !ENV &gsiberr ${DATA}/berror/gsi-coeffs-gfs-global.nc4
+#     gsi error covariance file: !ENV &gsiberr ${DATA}/berror/global_berror.f77
+      gsi berror namelist file: !ENV &gsibnml ${DATA}/berror/gfs_gsi_global.nml
+      processor layout x direction: &layout_gsib_x 3
+      processor layout y direction: &layout_gsib_y 2
+      debugging mode: false
+    saber outer blocks:
+    - saber block name: gsi interpolation to model grid
+      input variables: *control_vars
+      output variables: *control_vars
+      gsi akbk: *akbk
+      gsi error covariance file: *gsiberr
+      gsi berror namelist file: *gsibnml
+      processor layout x direction: *layout_gsib_x
+      processor layout y direction: *layout_gsib_y
+      debugging mode: false
+    linear variable change:
+      linear variable change name: Control2Analysis
+      input variables: *control_vars
+      output variables: &3dvars_anal [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr]
+  weight:
+    value: 0.125
+- covariance:
+    covariance model: ensemble
+    members from template:
+      template:
+        datetime: '{{BKG_ISOTIME}}'
+        filetype: fms restart
+        state variables: *3dvars_anal
+        datapath: ens/mem%mem%
+        filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc'
+        filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc'
+        filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc'
+        filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc'
+        filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res'
+      pattern: '%mem%'
+      nmembers: !ENV ${NMEM_ENKF}
+      zero padding: 3
+#    localization:
+#      localization method: SABER
+#      saber block:
+#      - saber block name: BUMP_NICAS
+#        input variables: *control_vars
+#        output variables: *control_vars
+#        active variables: *active_vars
+#        bump:
+#          datadir: *staticb_dir
+#          verbosity: main
+#          strategy: specific_univariate
+#          method: loc
+#          load_nicas_local: true
+#          grids:
+#          - prefix: nicas/nicas_3D_gfs
+#            variables: [stream_function,velocity_potential,air_temperature,relative_humidity,cloud_liquid_water,ozone_mass_mixing_ratio]
+  weight:
+    value: 0.875

parm/atm/lgetkf/lgetkf.yaml

@@ -22,7 +22,7 @@ background:
        state variables: [ua,va,t,DZ,delp,ps,sphum,ice_wat,liq_wat,o3mr,phis,
                         slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph,
                         u_srf,v_srf,f10m]
-       datapath: bkg/mem%mem%/RESTART
+       datapath: bkg/mem%mem%
        filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc'
        filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc'
        filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc'

parm/atm/obs/config/sondes.yaml

@@ -26,7 +26,7 @@ obs operator:
      - name: specificHumidity
    - name: SfcPCorrected
      da_psfc_scheme: GSI
-     geovar_sfc_geomz: surface_geometric_height
+     geovar_sfc_geomz: surface_geopotential_height
      geovar_geomz: geopotential_height
      variables: 
      - name: stationPressure
@@ -92,7 +92,7 @@ obs filters:
           error_min: 100.0         # 1 mb
           error_max: 300.0         # 3 mb
           geovar_geomz: geopotential_height
-          geovar_sfc_geomz: surface_geometric_height 
+          geovar_sfc_geomz: surface_geopotential_height 
   #
   # Gross error check with (O - B) / ObsError greater than threshold
   - filter: Background Check

test/atm/CMakeLists.txt

@@ -35,6 +35,11 @@ if (BUILD_GDASBUNDLE)
            COMMAND ${PROJECT_SOURCE_DIR}/test/atm/run_jedi_exe_3dvar.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
            WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
 
+  # test for ush/run_jedi_exe.py 3denvar
+  add_test(NAME test_gdasapp_run_jedi_exe_3denvar
+           COMMAND ${PROJECT_SOURCE_DIR}/test/atm/run_jedi_exe_3denvar.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
+           WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
+
   # test for ush/run_jedi_exe.py letkf
   add_test(NAME test_gdasapp_run_jedi_exe_letkf
            COMMAND ${PROJECT_SOURCE_DIR}/test/atm/run_jedi_exe_letkf.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}

test/atm/run_jedi_exe_3denvar.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+set -x
+
+bindir=$1
+srcdir=$2
+
+# Identify machine
+if [[ -d /scratch1 ]] ; then
+    machine="hera"
+elif [[ -d /work ]] ; then
+    machine="orion"
+else
+    echo "UNSUPPORTED MACHINE.  ABORT"
+    exit 99
+fi
+
+# Load modules
+set +x
+module use ${srcdir}/modulefiles
+module load GDAS/${machine}
+set -x
+module list
+
+# Set machine dependent variables
+if [ "$machine" = "hera" ] ; then
+    partition="hera"
+    gdasfix="/scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix"
+elif [ "$machine" = "orion" ]; then
+    partition="debug"
+    gdasfix="/work2/noaa/da/cmartin/GDASApp/fix"
+fi
+
+# Create test run directory
+mkdir -p ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_3denvar
+cd ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_3denvar
+
+# Create input yaml
+cat > ./3denvar_example.yaml << EOF
+working directory: ./
+GDASApp home: ${srcdir}
+GDASApp mode: variational
+template: ${srcdir}/parm/atm/variational/3dvar_dripcg.yaml
+config:
+  berror_yaml: ${srcdir}/parm/atm/berror/hybvar_gsibec.yaml
+  obs_dir: obs
+  diag_dir: diags
+  crtm_coeff_dir: crtm
+  bias_in_dir: obs
+  bias_out_dir: bc
+  obs_yaml_dir: ${srcdir}/parm/atm/obs/config
+  executable: ${bindir}/bin/fv3jedi_var.x
+  obs_list: ${srcdir}/parm/atm/obs/lists/gdas_prototype_3d.yaml
+  gdas_fix_root: ${gdasfix}
+  atm: true
+  layout_x: 1
+  layout_y: 1
+  atm_window_length: PT6H
+  valid_time: 2021-12-21T06:00:00Z
+  dump: gdas
+  case: C96
+  case_anl: C48
+  case_enkf: C48
+  staticb_type: gsibec
+  dohybvar: true
+  levs: 128
+  nmem: 10
+  interp_method: barycentric
+job options:
+  machine: ${machine}
+  account: da-cpu
+  queue: debug
+  partition: ${partition}
+  walltime: '30:00'
+  ntasks: 6
+  modulepath: ${srcdir}/modulefiles
+EOF
+
+# Execute run_jedi_exe.py
+if [ -e stdout.txt ]; then
+    rm -f stdout.txt
+fi
+${srcdir}/ush/run_jedi_exe.py -c ./3denvar_example.yaml > stdout.txt 2>&1
+rc=$?
+if [ $rc -ne 0 ]; then
+    exit $rc
+fi
+
+# Wait and ensure buffers flushed to disk
+sleep 5
+sync stdout.txt
+
+# Check for job submission error
+error=$(grep "sbatch: error" stdout.txt | wc -l)
+if [ $error -ne 0 ]; then
+    rc=$error
+    exit $rc
+fi
+
+# Cancel submitted job
+jobid=$(grep "Submitted" stdout.txt | awk -F' ' '{print $4}')
+scancel $jobid
+rc=$?
+if [ $rc -ne 0 ]; then
+    exit $rc
+fi
+
+# Check for valid yaml files
+ylist="3denvar_example.yaml gdas_variational.yaml"
+for yfile in $ylist; do
+    python3 -c 'import yaml, sys; yaml.safe_load(sys.stdin)' < $yfile
+    rc=$?
+    if [ $rc -ne 0 ]; then
+	exit $rc
+    fi
+done
+
+exit $rc

test/atm/run_jedi_exe_letkf.sh

@@ -23,11 +23,9 @@ module list
 
 # Set machine dependent variables
 if [ "$machine" = "hera" ] ; then
-    cominges="/scratch1/NCEPDEV/da/Russ.Treadon/GDASApp/cases"
     partition="hera"
     gdasfix="/scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix"
 elif [ "$machine" = "orion" ]; then
-    cominges="/work2/noaa/da/rtreadon/GDASApp/cases"
     partition="debug"
     gdasfix="/work2/noaa/da/cmartin/GDASApp/fix"
 fi
@@ -64,7 +62,6 @@ config:
   dohybvar: no
   levs: 128
   nmem: 5
-  comin_ges: ${cominges}
   interp_method: barycentric
 job options:
   machine: ${machine}

test/genYAML_prep.sh

@@ -19,6 +19,6 @@ config:
   valid_time: '2022-03-30T00:00:00Z'
   atm_window_length: PT6H
   CASE: 'C48'
-  CASE_ENKF: 'C48'
+  CASE_ANL: 'C48'
   LEVS: '128'
 EOF

test/genYAML_prep_aero.sh

@@ -19,6 +19,6 @@ config:
   valid_time: '2019-06-14T06:00:00Z'
   aero_window_length: PT6H
   CASE: 'C96'
-  CASE_ENKF: 'C48'
+  CASE_ANL: 'C48'
   LEVS: '128'
 EOF

test/genYAML_prep_land.sh

@@ -20,6 +20,6 @@ config:
   atm_window_length: PT6H
   land_window_length: PT6H
   CASE: 'C48'
-  CASE_ENKF: 'C48'
+  CASE_ANL: 'C48'
   LEVS: '128'
 EOF

ush/examples/run_jedi_exe/3denvar_hera.yaml

@@ -0,0 +1,37 @@
+working directory: /scratch2/NCEPDEV/stmp1/Cory.R.Martin/gdas_single_test_3denvar
+GDASApp home: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp
+GDASApp mode: variational
+template: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/variational/3dvar_dripcg.yaml
+config:
+  berror_yaml: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/berror/hybvar_gsibec.yaml
+  obs_dir: obs
+  diag_dir: diags
+  crtm_coeff_dir: crtm
+  bias_in_dir: obs
+  bias_out_dir: bc
+  obs_yaml_dir: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/config
+  executable: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/build/bin/fv3jedi_var.x
+  obs_list: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype_3d.yaml
+  gdas_fix_root: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix
+  atm: true
+  layout_x: 1
+  layout_y: 1
+  atm_window_length: PT6H
+  valid_time: 2021-12-21T06:00:00Z
+  dump: gdas
+  case: C96
+  case_anl: C48
+  case_enkf: C48
+  staticb_type: gsibec
+  dohybvar: true
+  levs: 128
+  nmem: 10
+  interp_method: barycentric
+job options:
+  machine: hera
+  account: da-cpu
+  queue: debug
+  partition: hera
+  walltime: '30:00'
+  ntasks: 6
+  modulepath: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/modulefiles

ush/examples/run_jedi_exe/3denvar_orion.yaml

@@ -0,0 +1,37 @@
+working directory: /work2/noaa/stmp/cmartin/gdas_single_test_3denvar
+GDASApp home: /work2/noaa/da/cmartin/GDASApp/work/GDASApp
+GDASApp mode: variational
+template: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/variational/3dvar_dripcg.yaml
+config:
+  berror_yaml: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/berror/hybvar_gsibec.yaml
+  obs_dir: obs
+  diag_dir: diags
+  crtm_coeff_dir: crtm
+  bias_in_dir: obs
+  bias_out_dir: bc
+  obs_yaml_dir: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/config
+  executable: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/build/bin/fv3jedi_var.x
+  obs_list: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype_3d.yaml
+  gdas_fix_root: /work2/noaa/da/cmartin/GDASApp/fix
+  atm: true
+  layout_x: 1
+  layout_y: 1
+  atm_window_length: PT6H
+  valid_time: 2021-12-21T06:00:00Z
+  dump: gdas
+  case: C96
+  case_anl: C48
+  case_enkf: C48
+  staticb_type: gsibec
+  dohybvar: true
+  levs: 128
+  nmem: 10
+  interp_method: barycentric
+job options:
+  machine: orion
+  account: da-cpu
+  queue: debug
+  partition: debug
+  walltime: '30:00'
+  ntasks: 6
+  modulepath: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/modulefiles

ush/examples/run_jedi_exe/4dhofx_hera.yaml

@@ -24,5 +24,5 @@ job options:
   partition: hera
   walltime: '30:00'
   ntasks: 6
-  ntasks-per-node: 2
+  ntasks-per-node: 1
   modulepath: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/modulefiles

ush/examples/run_jedi_exe/4dhofx_orion.yaml

@@ -24,5 +24,5 @@ job options:
   partition: debug
   walltime: '30:00'
   ntasks: 6
-  ntasks-per-node: 2
+  ntasks-per-node: 1
   modulepath: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/modulefiles

ush/examples/run_jedi_exe/letkf_hera.yaml

@@ -19,13 +19,9 @@ config:
   valid_time: 2021-12-21T06:00:00Z
   dump: gdas
   case: C48
-  case_anl: C48
   case_enkf: C48
-  staticb_type: gsibec
-  dohybvar: no
   levs: 128
   nmem: 10
-  comin_ges: /scratch1/NCEPDEV/da/Russ.Treadon/GDASApp/cases
   interp_method: barycentric
 job options:
   machine: hera