Add BUFR2IODA python API converter to prepoceanobs task

parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_argo.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "subpfl",
+  "subsets"          : "SUBPFL",
+  "source"           : "NCEP data tank",
+  "data_type"        : "argo",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ ARGO profiles",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "bathy",
+  "subsets"          : "BATHY",
+  "source"           : "NCEP data tank",
+  "data_type"        : "bathy",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ Bathythermal profiles",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_dbuoy.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "dbuoy",
+  "subsets"          : "DBUOY",
+  "source"           : "NCEP data tank",
+  "data_type"        : "dbuoy",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ drifting buoy (TAC:drifters, etc) profiles",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_dbuoyb.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "dbuoyb",
+  "subsets"          : "DBUOYB",
+  "source"           : "NCEP data tank",
+  "data_type"        : "dbuoyb",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ drifting buoy (BUFR:drifters, etc) profiles",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_glider.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "subpfl",
+  "subsets"          : "SUBPFL",
+  "source"           : "NCEP data tank",
+  "data_type"        : "glider",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ GLIDER profiles",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_marinemammals.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "tesac",
+  "subsets"          : "TESAC",
+  "source"           : "NCEP data tank",
+  "data_type"        : "marinemammals",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ Marine Mammals profiles",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_mbuoy.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "mbuoy",
+  "subsets"          : "MBUOY",
+  "source"           : "NCEP data tank",
+  "data_type"        : "mbuoy",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ moored buoy (TAC:Tropical Mooring) profiles",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_mbuoyb.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "mbuoyb",
+  "subsets"          : "MBUOYB",
+  "source"           : "NCEP data tank",
+  "data_type"        : "mbuoyb",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ  moored buoy (BUFR:Tropical Mooring) profiles",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "tesac",
+  "subsets"          : "TESAC",
+  "source"           : "NCEP data tank",
+  "data_type"        : "tesac",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ TESAC profiles",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_xbtctd.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "xbtctd",
+  "subsets"          : "XBTCTD",
+  "source"           : "NCEP data tank",
+  "data_type"        : "xbtctd",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ XBT/XCTD profiles",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_altkob.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "altkob",
+  "subsets"          : "ALTKOB",
+  "source"           : "NCEP data tank",
+  "data_type"        : "altkob",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ ALTKOB surface",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_ships.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "ships",
+  "subsets"          : "SHIPS",
+  "source"           : "NCEP data tank",
+  "data_type"        : "bathy",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ ship (TAC: restriced) surface",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_shipsu.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "shipsu",
+  "subsets"          : "SHIPSU",
+  "source"           : "NCEP data tank",
+  "data_type"        : "shipsu",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ ship (TAC: unrestricted) surface",
+  "data_provider"    : "U.S. NOAA"
+}

parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.json

@@ -0,0 +1,12 @@
+{
+  "data_format"      : "trkob",
+  "subsets"          : "TRACKOB",
+  "source"           : "NCEP data tank",
+  "data_type"        : "trackob",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ TRACKOB surface",
+  "data_provider"    : "U.S. NOAA"
+}

parm/soca/obs/config/dev/insitu_profile_argo.yaml

@@ -0,0 +1,22 @@
+obs space:
+  name: insitu_profile_argo
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_argo.${PDY}${cyc}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/diags/insitu_profile_argo.${PDY}${cyc}.nc4
+  simulated variables: [waterTemperature, salinity]
+  observed variables: [waterTemperature, salinity]
+  io pool:
+    max pool size: 1
+obs operator:
+  name: Composite
+  components:
+  - name: InsituTemperature
+    variables:
+    - name: waterTemperature
+obs error:
+  covariance model: diagonal

parm/soca/obs/config/dev/insitu_profile_bathy.yaml

@@ -0,0 +1,158 @@
+obs space:
+  name: insitu_profile_bathy
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_bathy.${PDY}${cyc}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/diags/insitu_profile_bathy.${PDY}${cyc}.nc4
+  simulated variables: [waterTemperature]
+  io pool:
+    max pool size: 1
+obs operator:
+  name: InsituTemperature
+obs error:
+  covariance model: diagonal
+
+#--------------------------------------------------------------------------------------
+#                   START OF OBS FILTERS
+# The QC filters used here are based on the document by IODE that can be found at
+# https://cdn.ioos.noaa.gov/media/2017/12/recommendations_in_situ_data_real_time_qc.pdf
+#--------------------------------------------------------------------------------------
+obs filters:
+
+  # Land Check
+  - filter: Domain Check
+    where:
+    - variable: {name: GeoVaLs/sea_area_fraction}
+      minvalue: 0.5
+#--------------------------------------------------------------------------------------
+## Filters for T:
+#--------------------------------------------------------------------------------------
+  #------------------------------------------------------------------------------------
+  ### Global range test
+  #------------------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: -2.5
+    maxvalue: 40.0
+
+  #------------------------------------------------------------------------------------
+  ### Regional range tests
+  #------------------------------------------------------------------------------------
+
+  #### Red Sea
+  #------------------------------------------------------------------------------------
+  ####
+  #### the document linked here describes Red sea as the are between 10N, 40E;
+  #### 20N, 50E; 30N, 30E; 10N, 40E. But that would also include Gulf of Aden.
+  #### A more reasonable choice seemed to be a box that extends from 10 N to
+  #### 30 N and 30 E to 45 East .
+  #------------------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: 21.7
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 10
+      maxvalue: 30
+    - variable:
+        name: MetaData/longitude
+      minvalue: 30
+
+  #### Mediterranean Sea
+  #-----------------------------------------------------------------------------
+  ##### Area 1/3 for Mediterranean Sea
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: 10.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 30
+      maxvalue: 40
+    - variable:
+        name: MetaData/longitude
+      minvalue: -6
+      maxvalue: 40
+  ##### Area 2/3 for Mediterranean Sea
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: 10.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 40
+      maxvalue: 41.5
+    - variable:
+        name: MetaData/longitude
+      minvalue: 20
+      maxvalue: 30
+  ##### Area 3/3 for Mediterranean Sea
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: 10.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 40
+      maxvalue: 46
+    - variable:
+        name: MetaData/longitude
+      minvalue: 0
+      maxvalue: 20
+
+  #### Northwestern shelves
+  #-----------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: -2.0
+    maxvalue: 24.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 50
+      maxvalue: 60
+    - variable:
+        name: MetaData/longitude
+      minvalue: -20
+      maxvalue: 10
+  #### Southwestern shelves
+  #-----------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: -2.0
+    maxvalue: 30
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 25
+      maxvalue: 50
+    - variable:
+        name: MetaData/longitude
+      minvalue: -30
+      maxvalue: 0
+
+  #### Arctic Sea
+  #-----------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: -1.92
+    maxvalue: 25.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 60
+
+  - filter: Background Check
+    filter variables: [{name: waterTemperature}]
+    threshold: 5.0
+    absolute threshold: 5.0
+

parm/soca/obs/config/insitu_profile_bathy.yaml → parm/soca/obs/config/dev/insitu_profile_dbuoy.yaml

@@ -1,13 +1,13 @@
 obs space:
-  name: insitu_profile_bathy
+  name: insitu_profile_dbuoy
   obsdatain:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_bathy.${PDY}${cyc}.nc4
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_dbuoy.${PDY}${cyc}.nc4
   obsdataout:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/diags/insitu_profile_bathy.${PDY}${cyc}.nc4
+      obsfile: !ENV ${DATA}/diags/insitu_profile_dbuoy.${PDY}${cyc}.nc4
   simulated variables: [waterTemperature, salinity]
   observed variables: [waterTemperature, salinity]
   io pool: