More user defined experiment parameters for the marine DA

parm/soca/berror/saber_block_ens.yaml

@@ -0,0 +1,12 @@
+covariance model: ensemble
+members from template:
+  template:
+    read_from_file: 1
+    date: '{{ATM_WINDOW_BEGIN}}'
+    basename: ./ens/
+    ocn_filename: MOM.res_%mem%.nc
+    ice_filename: cice.res_%mem%.nc
+    remap_filename: ./bkg/MOM.res.nc
+    state variables: [tocn, sonc, ssh, hocn, cicen, hicen]
+  pattern: '%mem%'
+  nmembers: 3

parm/soca/berror/soca_bump2d.yaml

@@ -23,8 +23,8 @@ bump:
   model:
     do not cross mask boundaries: false
   nicas:
-    resolution: 1.0                   # Do not use these numbers for
-    max horizontal grid size: 15000   # science experiments, only good for testing
+    resolution: !ENV ${NICAS_RESOL}
+    max horizontal grid size: ${NICAS_GRID_SIZE}
   grids:
   - model:
       variables:

parm/soca/berror/soca_bump_split.yaml

@@ -24,8 +24,8 @@ bump:
   model:
     do not cross mask boundaries: false
   nicas:
-    resolution: 1.0                   # Do not use these numbers for
-    max horizontal grid size: 15000   # science experiments, only good for testing
+    resolution: !ENV ${NICAS_RESOL}
+    max horizontal grid size: ${NICAS_GRID_SIZE}
   grids:
   - model:
       variables:

parm/soca/obs/config/obs_list.yaml

@@ -0,0 +1 @@
+../obs_list.yaml

parm/soca/variational/3dvarfgat.yaml

@@ -22,7 +22,7 @@ cost function:
     state variables: *model_vars
   background error: !INC ${SABER_BLOCKS_YAML}
 
-  observations: !INC ${OBS_YAML}
+  observations: !INC ${OBS_LIST}
 
 variational:
   minimizer:

scripts/exgdas_global_marine_analysis_prep.py

@@ -174,7 +174,7 @@ def gen_bkg_list(bkg_path, out_path, window_begin=' ', file_type='gdas.t*.ocnf00
 ufsda.r2d2.setup(r2d2_config_yaml=os.path.join(anl_dir, 'r2d2_config.yaml'), shared_root=comin_obs)
 
 # create config dict from runtime env
-envconfig = ufsda.misc_utils.get_env_config(component='notatm')
+envconfig = ufsda.misc_utils.get_env_config(component='soca')
 stage_cfg = YAMLFile(path=os.path.join(gdas_home,
                                        'parm',
                                        'templates',
@@ -243,8 +243,6 @@ def gen_bkg_list(bkg_path, out_path, window_begin=' ', file_type='gdas.t*.ocnf00
                                   'berror',
                                   'soca_bump2d.yaml')
 config = YAMLFile(path=bump_yaml_template)
-config = Template.substitute_structure(config, TemplateConstants.DOUBLE_CURLY_BRACES, {'datadir': bumpdir}.get)
-config = Template.substitute_structure(config, TemplateConstants.DOLLAR_PARENTHESES, {'datadir': bumpdir}.get)
 config = Template.substitute_structure(config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig.get)
 config = Template.substitute_structure(config, TemplateConstants.DOLLAR_PARENTHESES, envconfig.get)
 config.save(bump_yaml)
@@ -286,7 +284,15 @@ def gen_bkg_list(bkg_path, out_path, window_begin=' ', file_type='gdas.t*.ocnf00
              window_begin=window_begin,
              yaml_name='bkg_list.yaml')
 os.environ['BKG_LIST'] = 'bkg_list.yaml'
-os.environ['SABER_BLOCKS_YAML'] = os.path.join(gdas_home, 'parm', 'soca', 'berror', 'saber_blocks.yaml')
+
+# select the SABER BLOCKS to use
+if 'SABER_BLOCKS_YAML' in os.environ and os.environ['SABER_BLOCKS_YAML']:
+    saber_blocks_yaml = os.getenv('SABER_BLOCKS_YAML')
+    logging.info(f"using non-default SABER blocks yaml: {saber_blocks_yaml}")
+else:
+    logging.info(f"using default SABER blocks yaml")
+    os.environ['SABER_BLOCKS_YAML'] = os.path.join(gdas_home, 'parm', 'soca', 'berror', 'saber_blocks.yaml')
+
 logging.info(f"{config}")
 varconfig = YAMLFile(path=var_yaml_template)
 varconfig = Template.substitute_structure(varconfig, TemplateConstants.DOUBLE_CURLY_BRACES, config.get)

test/soca/gw/CMakeLists.txt

@@ -22,10 +22,15 @@ set_tests_properties(test_gdasapp_soca_jjob_soca_prep
   PROPERTIES
     DEPENDS "test_gdasapp_soca_cycled_exp;test_gdasapp_soca_bkg2comrot")
 
-add_test(NAME test_gdasapp_soca_jjob_soca_run
-  COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/jjob_run.sh
-          ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
-  WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun)
-  set_tests_properties(test_gdasapp_soca_jjob_soca_run
-  PROPERTIES
-    DEPENDS "test_gdasapp_soca_cycled_exp;test_gdasapp_soca_bkg2comrot;test_gdasapp_soca_jjob_soca_prep")
+# TODO: This test is broken on Orion, skip it for now when running the CI on Orion.
+IF (IS_DIRECTORY /work2/noaa/da)
+  message("Skipping this test for now")
+ELSE()
+  add_test(NAME test_gdasapp_soca_jjob_soca_run
+    COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/jjob_run.sh
+            ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
+    WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun)
+    set_tests_properties(test_gdasapp_soca_jjob_soca_run
+    PROPERTIES
+      DEPENDS "test_gdasapp_soca_cycled_exp;test_gdasapp_soca_bkg2comrot;test_gdasapp_soca_jjob_soca_prep")
+ENDIF()

test/soca/gw/config.yaml

@@ -2,6 +2,10 @@ ocnanal:
   SOCA_INPUT_FIX_DIR: '@SOCA_INPUT_FIX_DIR@'
   CASE_ANL: 'C24'
   COMIN_OBS: '@COMIN_OBS@'
-  SOCA_NINNER: 3
+  SOCA_OBS_LIST: '@SOCA_OBS_LIST@'
+  SOCA_NINNER: 1
   R2D2_OBS_SRC: 'gdasapp'
   R2D2_OBS_DUMP: 'soca'
+  SABER_BLOCKS_YAML: ''
+  NICAS_RESOL: 1
+  NICAS_GRID_SIZE: 15000

test/soca/gw/jjob_prep.sh

@@ -6,13 +6,25 @@ srcdir=$2
 # export env. var.
 source "${srcdir}/test/soca/gw/runtime_vars.sh" "${bindir}" "${srcdir}"
 
-OCNDIR="${ROTDIR}/${PSLOT}/gdas.${PDY}/${gcyc}/ocean/" 
+OCNDIR="${ROTDIR}/${PSLOT}/gdas.${PDY}/${gcyc}/ocean/"
 
 rm -r ${OCNDIR}
 
 # prepare background from previous cycle
 mkdir -p ${OCNDIR}
 cp -r "${bindir}/test/soca/bkg/"* ${OCNDIR}
 
+# detemine machine from config.base
+machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"')
+
 # run prep jjob
-"${HOMEgfs}/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP"
+if [[ ${machine} == 'CONTAINER' ]]; then
+    "${HOMEgfs}/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP"
+else
+    sbatch --ntasks=1 \
+           --account=da-cpu \
+           --qos=debug \
+           --time=00:5:00 \
+           --export=ALL \
+           --wait "${HOMEgfs}/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP"
+fi

test/soca/gw/jjob_run.sh

@@ -5,5 +5,17 @@ srcdir=$2
 # export env. var.
 source "${srcdir}/test/soca/gw/runtime_vars.sh" "${bindir}" "${srcdir}"
 
+# detemine machine from config.base
+machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"')
+
 # run DA jjob
-"${HOMEgfs}/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN"
+if [[ ${machine} == 'CONTAINER' ]]; then
+    "${HOMEgfs}/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN"
+else
+    sbatch --ntasks=16 \
+           --account=da-cpu \
+           --qos=debug \
+           --time=00:10:00 \
+           --export=ALL \
+           --wait "${HOMEgfs}/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN"
+fi

test/soca/gw/runtime_vars.sh

@@ -9,6 +9,7 @@ export cyc=12
 export gcyc=06
 export ROTDIR="${bindir}/test/soca/gw/testrun/ROTDIRS"
 export DATAROOT="${bindir}/test/soca/gw/testrun/RUNDIRS/gdas_test"
+export COMROOT=${DATAROOT}
 export COMIN_GES="${bindir}/test/soca/bkg"
 export PSLOT='gdas_test'
 export EXPDIRS="${bindir}/test/soca/gw/testrun/experiments/"

test/soca/gw/setup_workflow_exp.sh

@@ -32,8 +32,10 @@ cp -r "${configdir}" config
 cp "${srcdir}/test/soca/gw/config.yaml" .
 soca_input_fix_dir="${bindir}/soca_static"
 comin_obs="${bindir}/test/soca/obs/r2d2-shared"
+soca_obs_list="${srcdir}/parm/soca/obs/obs_list.yaml"
 sed -i -e "s~@SOCA_INPUT_FIX_DIR@~${soca_input_fix_dir}~g" config.yaml
 sed -i -e "s~@COMIN_OBS@~${comin_obs}~g" config.yaml
+sed -i -e "s~@SOCA_OBS_LIST@~${soca_obs_list}~g" config.yaml
 
 echo "Running global-workflow experiment generation script"
 ./setup_expt.py cycled --idate "${idate}"  \

test/soca/runtime_vars.sh

@@ -46,4 +46,6 @@ export R2D2_OBS_DUMP=soca
 export R2D2_OBS_SRC=gdasapp
 export R2D2_OBS_WINDOW=24     # R2D2 sampling DB window
 export SOCA_NINNER=2
+export NICAS_RESOL=1
+export NICAS_GRID_SIZE=15000
 export APRUN_OCNANAL="$MPIEXEC_EXEC $MPIEXEC_NPROC 2"

ush/soca/examples/run_jjobs_container.yaml

@@ -22,9 +22,9 @@ gw environement:
     COMIN_GES: /home/gvernier/wrk/gw/sorc/gdas.cd/build/test/soca/bkg
 
   working directories:
-    ROTDIR: /home/gvernier/wrk/gw/sorc/gdas.cd/ush/runjjobs/test/ROTDIRS
-    EXPDIRS: /home/gvernier/wrk/gw/sorc/gdas.cd/ush/runjjobs/test/experiments
-    STMP: /home/gvernier/wrk/gw/sorc/gdas.cd/ush/runjjobs/test
+    ROTDIR: /home/gvernier/wrk/gw/sorc/gdas.cd/ush/soca/test/ROTDIRS
+    EXPDIRS: /home/gvernier/wrk/gw/sorc/gdas.cd/ush/soca/test/experiments
+    STMP: /home/gvernier/wrk/gw/sorc/gdas.cd/ush/soca/test
 
   jedi:
     OOPS_TRACE: 1
@@ -33,9 +33,13 @@ gw environement:
 
 setup_expt config:
   ocnanal:
-    SOCA_INPUT_FIX_DIR: '/home/gvernier/wrk/gw/sorc/gdas.cd/build/soca_static'
-    CASE_ANL: 'C48'
-    COMIN_OBS: '/home/gvernier/wrk/gw/sorc/gdas.cd/build/test/soca/obs/r2d2-shared'
+    SOCA_INPUT_FIX_DIR: /home/gvernier/wrk/gw/sorc/gdas.cd/build/soca_static
+    CASE_ANL: C48
+    SOCA_OBS_LIST: /home/gvernier/wrk/gw/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
+    COMIN_OBS: /home/gvernier/wrk/gw/sorc/gdas.cd/build/test/soca/obs/r2d2-shared
     SOCA_NINNER: 1
-    R2D2_OBS_SRC: 'gdasapp'
-    R2D2_OBS_DUMP: 'soca'
+    R2D2_OBS_SRC: gdasapp
+    R2D2_OBS_DUMP: soca
+    SABER_BLOCKS_YAML: /home/gvernier/wrk/gw/sorc/gdas.cd/parm/soca/berror/saber_blocks.yaml
+    NICAS_RESOL: 1
+    NICAS_GRID_SIZE: 15000

ush/soca/examples/run_jjobs_hera.yaml

@@ -34,10 +34,14 @@ setup_expt config:
   ocnanal:
     SOCA_INPUT_FIX_DIR: '/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca'
     CASE_ANL: 'C48'
+    SOCA_OBS_LIST: ''
     COMIN_OBS:  /scratch2/NCEPDEV/marineda/r2d2
     SOCA_NINNER: 3
     R2D2_OBS_SRC: 'gdas_marine'
     R2D2_OBS_DUMP: 's2s_v1'
+    SABER_BLOCKS_YAML: ''
+    NICAS_RESOL: 1
+    NICAS_GRID_SIZE: 15000
 
 job options:
   account: marine-cpu

ush/soca/examples/run_jjobs_hera_025.yaml

@@ -34,10 +34,14 @@ setup_expt config:
   ocnanal:
     SOCA_INPUT_FIX_DIR: '/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/1440x1080x75/soca'
     CASE_ANL: 'C384'
+    SOCA_OBS_LIST: ''
     COMIN_OBS:  /scratch2/NCEPDEV/marineda/r2d2
-    SOCA_NINNER: 3
+    SOCA_NINNER: 1
     R2D2_OBS_SRC: 'gdas_marine'
     R2D2_OBS_DUMP: 's2s_v1'
+    SABER_BLOCKS_YAML: ''
+    NICAS_RESOL: 1
+    NICAS_GRID_SIZE: 15000
 
 job options:
   account: marine-cpu