Incorporate multi-cycle concatination and separation of variables in post-BUFR observations

parm/ioda/bufr2ioda/bufr2ioda_insitu_salt_profile_argo.yaml

@@ -0,0 +1,11 @@
+cycle_datetime: '{{ current_cycle | to_YMDH }}'
+cycle_type: '{{ RUN }}'
+data_description: 6-hrly in situ ARGO profiles
+data_format: subpfl
+data_provider: U.S. NOAA
+data_type: argo
+dump_directory: '{{ DMPDIR }}'
+ioda_directory: '{{ COM_OBS }}'
+source: NCEP data tank
+subsets: SUBPFL
+ocean_basin: '{{ OCEAN_BASIN_FILE }}'

parm/ioda/bufr2ioda/bufr2ioda_insitu_temp_profile_argo.yaml

@@ -0,0 +1,11 @@
+cycle_datetime: '{{ current_cycle | to_YMDH }}'
+cycle_type: '{{ RUN }}'
+data_description: 6-hrly in situ ARGO profiles
+data_format: subpfl
+data_provider: U.S. NOAA
+data_type: argo
+dump_directory: '{{ DMPDIR }}'
+ioda_directory: '{{ COM_OBS }}'
+source: NCEP data tank
+subsets: SUBPFL
+ocean_basin: '{{ OCEAN_BASIN_FILE }}'

parm/soca/obs/config/insitu_salt_profile_argo.yaml

@@ -0,0 +1,179 @@
+obs space:
+  name: insitu_salt_profile_argo
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: ${DATA}/obs/${OPREFIX}insitu_salt_profile_argo.${PDY}${cyc}.nc4
+    obsgrouping:
+      group variables: [latitude, longitude, dateTime]
+      sort variable: depth
+      sort group: MetaData
+      sort order: ascending
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: ${DATA}/diags/insitu_salt_profile_argo.${PDY}${cyc}.nc4
+  simulated variables: [salinity]
+  observed variables: [salinity]
+  io pool:
+    max pool size: 1
+obs operator:
+  name: VertInterp
+  observation alias file: ./obsop_name_map.yaml
+  vertical coordinate: sea_water_depth
+  observation vertical coordinate: depth
+  interpolation method: linear
+obs error:
+  covariance model: diagonal
+
+#-------------------------------------------------------------------------------
+#                   START OF OBS FILTERS (work done by Kriti Bhargava)
+# The QC filters used here are based on the document by IODE that can be found at
+# https://cdn.ioos.noaa.gov/media/2017/12/recommendations_in_situ_data_real_time_qc.pdf
+#-------------------------------------------------------------------------------
+obs filters:
+
+  # land check
+  - filter: Domain Check
+    where:
+    - variable: {name: GeoVaLs/sea_area_fraction}
+      minvalue: 0.5
+
+
+#-------------------------------------------------------------------------------
+## Filters for S:
+#-------------------------------------------------------------------------------
+  #-----------------------------------------------------------------------------
+  ### Global range test
+  #-----------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: salinity}]
+    minvalue: 2.0
+    maxvalue: 41.0
+
+  #-----------------------------------------------------------------------------
+  ### Regional range test
+  #-----------------------------------------------------------------------------
+  #### Red Sea
+  #-----------------------------------------------------------------------------
+  ####
+  #### the document linked here describes Red sea as the are between 10N, 40E;
+  #### 20N, 50E; 30N, 30E; 10N, 40E. But that would also include Gulf of Aden.
+  #### A more reasonable choice seemed to be a box that extends from 10 N to
+  #### 30 N and 30 E to 45 East .
+
+  - filter: Bounds Check
+    filter variables: [{name: salinity}]
+    minvalue: 2.0
+    maxvalue: 41.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 10
+      maxvalue: 30
+    - variable:
+        name: MetaData/longitude
+      minvalue: 30
+      maxvalue: 45
+
+  #### Mediterranean Sea
+  #-----------------------------------------------------------------------------
+  ##### Area 1/3 for Mediterranean Sea
+  - filter: Bounds Check
+    filter variables: [{name: salinity}]
+    minvalue: 2.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 30
+      maxvalue: 40
+    - variable:
+        name: MetaData/longitude
+      minvalue: -6
+      maxvalue: 40
+  ##### Area 2/3 for Mediterranean Sea
+  - filter: Bounds Check
+    filter variables: [{name: salinity}]
+    minvalue: 2.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 40
+      maxvalue: 41.5
+    - variable:
+        name: MetaData/longitude
+      minvalue: 20
+      maxvalue: 30
+  ##### Area 3/3 for Mediterranean Sea
+  - filter: Bounds Check
+    filter variables: [{name: salinity}]
+    minvalue: 2.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 40
+      maxvalue: 46
+    - variable:
+        name: MetaData/longitude
+      minvalue: 0
+      maxvalue: 20
+
+
+  #### Northwestern shelves
+  #-----------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: salinity}]
+    minvalue: 0.0
+    maxvalue: 37.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 50
+      maxvalue: 60
+    - variable:
+        name: MetaData/longitude
+      minvalue: -20
+      maxvalue: 10
+
+  #### Southwestern shelves
+  #-----------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: salinity}]
+    minvalue: 0.0
+    maxvalue: 38
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 25
+      maxvalue: 50
+    - variable:
+        name: MetaData/longitude
+      minvalue: -30
+      maxvalue: 0
+
+  #### Arctic Sea
+  #-----------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: salinity}]
+    minvalue: 2.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 60
+
+  - filter: Background Check
+    filter variables: [{name: salinity}]
+    threshold: 5.0
+    absolute threshold: 5.0
+
+obs localizations:
+- localization method: Rossby
+  base value: 100.0e3
+  rossby mult: 1.0
+  min grid mult: 2.0
+  min value: 200.0e3
+  max value: 900.0e3

parm/soca/obs/config/insitu_temp_profile_argo.yaml

@@ -0,0 +1,173 @@
+obs space:
+  name: insitu_temp_profile_argo
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: ${DATA}/obs/${OPREFIX}insitu_temp_profile_argo.${PDY}${cyc}.nc4
+    obsgrouping:
+      group variables: [latitude, longitude, dateTime]
+      sort variable: depth
+      sort group: MetaData
+      sort order: ascending
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: ${DATA}/diags/insitu_temp_profile_argo.${PDY}${cyc}.nc4
+  simulated variables: [waterTemperature]
+  observed variables: [waterTemperature]
+  io pool:
+    max pool size: 1
+obs operator:
+  name: InsituTemperature
+obs error:
+  covariance model: diagonal
+
+#-------------------------------------------------------------------------------
+#                   START OF OBS FILTERS (work done by Kriti Bhargava)
+# The QC filters used here are based on the document by IODE that can be found at
+# https://cdn.ioos.noaa.gov/media/2017/12/recommendations_in_situ_data_real_time_qc.pdf
+#-------------------------------------------------------------------------------
+obs filters:
+
+  # land check
+  - filter: Domain Check
+    where:
+    - variable: {name: GeoVaLs/sea_area_fraction}
+      minvalue: 0.5
+
+#-------------------------------------------------------------------------------
+## Filters for T:
+#-------------------------------------------------------------------------------
+  #-------------------------------------------------------------------------------
+  ### Global range test
+  #-----------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: -2.5
+    maxvalue: 40.0
+
+  #-----------------------------------------------------------------------------
+  ### Regional range tests
+  #-----------------------------------------------------------------------------
+
+  #### Red Sea
+  #-----------------------------------------------------------------------------
+  ####
+  #### the document linked here describes Red sea as the are between 10N, 40E;
+  #### 20N, 50E; 30N, 30E; 10N, 40E. But that would also include Gulf of Aden.
+  #### A more reasonable choice seemed to be a box that extends from 10 N to
+  #### 30 N and 30 E to 45 East .
+
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: 21.7
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 10
+      maxvalue: 30
+    - variable:
+        name: MetaData/longitude
+      minvalue: 30
+      maxvalue: 45
+
+  #### Mediterranean Sea
+  #-----------------------------------------------------------------------------
+  ##### Area 1/3 for Mediterranean Sea
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: 10.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 30
+      maxvalue: 40
+    - variable:
+        name: MetaData/longitude
+      minvalue: -6
+      maxvalue: 40
+  ##### Area 2/3 for Mediterranean Sea
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: 10.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 40
+      maxvalue: 41.5
+    - variable:
+        name: MetaData/longitude
+      minvalue: 20
+      maxvalue: 30
+  ##### Area 3/3 for Mediterranean Sea
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: 10.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 40
+      maxvalue: 46
+    - variable:
+        name: MetaData/longitude
+      minvalue: 0
+      maxvalue: 20
+
+  #### Northwestern shelves
+  #-----------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: -2.0
+    maxvalue: 24.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 50
+      maxvalue: 60
+    - variable:
+        name: MetaData/longitude
+      minvalue: -20
+      maxvalue: 10
+  #### Southwestern shelves
+  #-----------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: -2.0
+    maxvalue: 30
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 25
+      maxvalue: 50
+    - variable:
+        name: MetaData/longitude
+      minvalue: -30
+      maxvalue: 0
+
+  #### Arctic Sea
+  #-----------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: -1.92
+    maxvalue: 25.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 60
+
+  - filter: Background Check
+    filter variables: [{name: waterTemperature}]
+    threshold: 5.0
+    absolute threshold: 5.0
+
+obs localizations:
+- localization method: Rossby
+  base value: 100.0e3
+  rossby mult: 1.0
+  min grid mult: 2.0
+  min value: 200.0e3
+  max value: 900.0e3

parm/soca/obs/obs_list.yaml

@@ -26,7 +26,9 @@ observers:
 
 # in situ: monthly
 #- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_bathy.yaml
-- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_argo.yaml
+#- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_argo.yaml
+#- !INC ${MARINE_OBS_YAML_DIR}/insitu_temp_profile_argo.yaml
+#- !INC ${MARINE_OBS_YAML_DIR}/insitu_salt_profile_argo.yaml
 #- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_glider.yaml
 #- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_tesac.yaml
 #- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_tesac_salinity.yaml