Add BUFR2IODA python API converter to prepoceanobs task

parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "bathy",
+  "subsets"          : "BATHY",
+  "source"           : "NCEP data tank",
+  "data_type"        : "bathy",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ Bathythermal profiles",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "tesac",
+  "subsets"          : "TESAC",
+  "source"           : "NCEP data tank",
+  "data_type"        : "tesac",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ TESAC profiles",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "trkob",
+  "subsets"          : "TRACKOB",
+  "source"           : "NCEP data tank",
+  "data_type"        : "trackob",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ TRACKOB surface",
+  "data_provider"    : "U.S. NOAA"
+}

parm/soca/obsprep/obsprep_config.yaml

@@ -153,17 +153,17 @@ observations:
     name: insitu_profile_bathy
     provider: GTS
     dmpdir subdir: atmos
-    output file: '*.bathy_profile.ioda.nc'
-    dmpdir regex: 'bathy.*.dcom_subsampled'
+    output file: 'bathy_profile.ioda.nc'
+    dmpdir regex: 'gdas.*.bathy.*.bufr_d'
 - obs space:
     name: insitu_profile_tesac
     provider: GTS
     dmpdir subdir: atmos
-    output file: '*.tesac_profile.ioda.nc'
-    dmpdir regex: 'tesac.*.dcom_subsampled'
+    output file: 'tesac_profile.ioda.nc'
+    dmpdir regex: 'gdas.*.tesac.*.bufr_d'
 - obs space:
     name: insitu_surface_trkob
     provider: GTS
     dmpdir subdir: atmos
-    output file: '*.trkob_surface.ioda.nc'
-    dmpdir regex: 'trkob.*.dcom_subsampled'
+    output file: 'trkob_surface.ioda.nc'
+    dmpdir regex: 'gdas.*.trkob.*.bufr_d'

scripts/exglobal_prep_ocean_obs.py

@@ -4,14 +4,19 @@
 from datetime import datetime, timedelta
 from multiprocessing import Process
 import os
+from pathlib import Path
 import subprocess
 from soca import prep_marine_obs
 from wxflow import YAMLFile, save_as_yaml, FileHandler, Logger
+from wxflow import add_to_datetime, to_timedelta, datetime_to_YMDH
+from gen_bufr2ioda_json import gen_bufr_json
 
 logger = Logger()
 
 cyc = os.getenv('cyc')
 PDY = os.getenv('PDY')
+RUN = os.getenv('RUN')
+COMIN_OBS = os.getenv('COMIN_OBS')
 
 # Set the window times
 cdateDatetime = datetime.strptime(PDY + cyc, '%Y%m%d%H')
@@ -29,12 +34,49 @@
 
 OBSPREP_YAML = os.getenv('OBSPREP_YAML')
 
+# BUFR2IODA json and python scripts
+JSON_TMPL_DIR = os.getenv('JSON_TMPL_DIR')
+BUFR2IODA_PY_DIR = os.getenv('BUFR2IODA_PY_DIR')
+
 if os.path.exists(OBSPREP_YAML):
     obsprepConfig = YAMLFile(OBSPREP_YAML)
 else:
     logger.critical(f"OBSPREP_YAML file {OBSPREP_YAML} does not exist")
     raise FileNotFoundError
 
+if not os.path.exists(COMOUT_OBS):
+    os.makedirs(COMOUT_OBS)
+
+
+def bufr2ioda(obtype, PDY, cyc, RUN, COMIN_OBS, COMOUT_OBS):
+    logger.info(f"Process {obtype} for {RUN}.{PDY}/{cyc} from {COMIN_OBS} to {COMOUT_OBS}")
+
+    # Load configuration
+    config = {
+        'RUN': RUN,
+        'current_cycle': cdateDatetime,
+        'DMPDIR': COMIN_OBS,
+        'COM_OBS': COMOUT_OBS,
+    }
+
+    json_output_file = os.path.join(COMOUT_OBS, f"{obtype}_{datetime_to_YMDH(cdateDatetime)}.json")
+    filename = 'bufr2ioda_' + obtype + '.json'
+    template = os.path.join(JSON_TMPL_DIR, filename)
+
+    # Generate cycle specific json from TEMPLATE
+    gen_bufr_json(config, template, json_output_file)
+
+    bufr2iodapy = BUFR2IODA_PY_DIR + '/bufr2ioda_' + obtype + '.py'
+    logger.info(f"BUFR2IODA python scripts: {bufr2iodapy}")
+
+    try:
+        subprocess.run(['python', bufr2iodapy, '-c', json_output_file, '-v'])
+        logger.info(f"BUFR2IODA python API converter on obs space {obtype} ran successfully")
+    except subprocess.CalledProcessError as e:
+        logger.info(f"BUFR2IODA python API converter failed with error {e}, \
+            return code {e.returncode}")
+        return e.returncode
+
 
 def run_netcdf_to_ioda(obsspace_to_convert):
     name, iodaYamlFilename = obsspace_to_convert
@@ -94,9 +136,8 @@ def run_netcdf_to_ioda(obsspace_to_convert):
                 outputFilename = f"gdas.t{cyc}z.{obs_space_name}.{PDY}{cyc}.nc4"
                 obsprepSpace['output file'] = outputFilename
 
-                # Skip in situ IODA conversion for now
                 if obsprepSpaceName.split('_')[0] == 'insitu':
-                    logger.info("Skipping insitu conversion for now")
+                    bufr2ioda(obsprepSpaceName, PDY, cyc, RUN, COMIN_OBS, COMOUT_OBS)
                 else:
                     iodaYamlFilename = obsprepSpaceName + '2ioda.yaml'
                     save_as_yaml(obsprepSpace, iodaYamlFilename)
@@ -122,7 +163,4 @@ def run_netcdf_to_ioda(obsspace_to_convert):
 for process in processes:
     process.join()
 
-if not os.path.exists(COMOUT_OBS):
-    os.makedirs(COMOUT_OBS)
-
 FileHandler({'copy': files_to_save}).sync()

test/soca/gw/setup_obsprep.sh

@@ -41,11 +41,12 @@ for PDY in "${PDYs[@]}"; do
                 done
             else
                 # Copy subsampled monthly in situ BUFR
-                # Example filename: tesac.201804.dcom_subsampled
+                # Example filename: tesac.201804.dcom_subsampled have been copied as
+                # gdas.t00z.tesac.tm00.bufr_d (the original filename)
                 # TODO: SP to replace these with daily and monthly subsampled with proper dates 
                 fullsubdir="$PDYdir/$cyc/atmos"
                 mkdir -p "$fullsubdir"
-                for file in "$testdatadir_bufr/*.${PDY:0:6}.dcom_subsampled"; do
+                for file in "$testdatadir_bufr/*.bufr_d"; do
                     cp -p $file $fullsubdir
                 done
             fi