Converts prepoceanobs to use JCB-style obs list and config files

parm/soca/obs/obs_list.yaml.j2

@@ -0,0 +1,50 @@
+observations:
+# ADT
+- adt_rads_all
+
+#SSS
+- sss_smap_l2
+- sss_smos_l2
+
+# SST
+- sst_avhrr_ma_l3u
+- sst_avhrr_mb_l3u
+- sst_avhrr_mc_l3u
+- sst_viirs_npp_l3u
+- sst_viirs_n20_l3u
+- sst_abi_g16_l3c
+- sst_abi_g17_l3c
+- sst_ahi_h08_l3c
+
+# Ice concentration
+- icec_amsr2_north
+- icec_amsr2_south
+#- icec_viirs_n20_l2_north
+#- icec_viirs_n20_l2_south
+#- icec_amsu_mb_l2
+#- icec_atms_n20_l2
+#- icec_atms_n21_l2
+#- icec_atms_npp_l2
+#- icec_gmi_gpm_l2
+#- icec_ssmis_f17_l2
+
+# in situ: monthly
+#- insitu_profile_bathy
+#- insitu_temp_profile_argo
+#- insitu_salt_profile_argo
+#- insitu_profile_glider
+#- insitu_profile_tesac
+#- insitu_profile_tesac_salinity
+#- insitu_profile_marinemammal
+#- insitu_profile_xbtctd
+#- insitu_surface_altkob
+#- insitu_surface_trkob
+#- insitu_surface_trkob_salinity
+
+# in situ: daily
+#- insitu_profile_dbuoy
+#- insitu_profile_dbuoyb
+#- insitu_profile_mbuoy
+#- insitu_profile_mbuoyb
+- insitu_profile_tropical
+- insitu_surface_drifter

parm/soca/obs/obs_list_base_yaml.j2

@@ -0,0 +1,13 @@
+# used with obs list to load obs configs
+
+    observers:
+{% for observation_from_jcb in observations %}
+{% if use_observer(observation_from_jcb) %}
+{% filter indent(width=4) %}
+{% set file_path = app_path_observations + '/' + observation_from_jcb + '.yaml.j2' %}
+{% include file_path %}
+{% endfilter %}
+{% endif %}
+{% endfor %}
+
+

ush/soca/prep_ocean_obs.py

@@ -82,8 +82,12 @@ def initialize(self):
         except OSError:
             logger.warning("Could not copy RECCAP2_region_masks_all_v20221025.nc")
 
-        OBS_YAML = self.task_config['OBS_YAML']
-        observer_config = YAMLFile(OBS_YAML)
+        self.task_config.app_path_observations = os.path.join(self.task_config['PARMgfs'], 'gdas/jcb-gdas/observations/marine')
+        OBS_YAML = os.path.join(self.task_config['PARMgfs'], 'gdas/soca/obs/obs_list.yaml.j2')
+        self.task_config.observations = parse_j2yaml(OBS_YAML, self.task_config)['observations']
+
+        obsconfigfile = os.path.join(self.task_config['PARMgfs'], 'gdas/soca/obs/obs_list_base_yaml.j2')
+        obsconfig = parse_j2yaml(obsconfigfile, self.task_config)['observers']
 
         OBSPREP_YAML = self.task_config['OBSPREP_YAML']
         if os.path.exists(OBSPREP_YAML):
@@ -108,23 +112,17 @@ def initialize(self):
 
         try:
             # go through the sources in OBS_YAML
-            for observer in observer_config['observers']:
-                try:
-                    obs_space_name = observer['obs space']['name']
-                    logger.info(f"Trying to find observation {obs_space_name} in OBSPREP_YAML")
-                except KeyError:
-                    logger.warning("Ill-formed observer yaml file, skipping")
-                    continue
+            for observation in obsconfig:
+                obs_space = observation['obs space']
+                obs_space_name = obs_space['name']
 
                 # find match to the obs space from OBS_YAML in OBSPREP_YAML
                 # this is awkward and unpythonic, so feel free to improve
                 for obsprep_entry in obsprep_config['observations']:
-                    obsprep_space = obsprep_entry['obs space']
+                    obsprep_space = obsprep_entry['obs space']  # the whole thing is needed later
                     obsprep_space_name = obsprep_space['name']
-
                     if obsprep_space_name == obs_space_name:
-                        obtype = obsprep_space_name  # for brevity
-                        logger.info(f"Observer {obtype} found in OBSPREP_YAML")
+                        logger.info(f"Observer {obs_space_name} found in OBSPREP_YAML")
 
                         try:
                             obs_window_back = obsprep_space['window']['back']
@@ -145,17 +143,18 @@ def initialize(self):
                                                                        window_cdates)
 
                         if not fetched_files:
-                            logger.warning(f"No files found for obs source {obtype}, skipping")
-                            break  # go to next observer in OBS_YAML
+                            logger.warning(f"No files found for obs source {obs_space_name}, skipping")
+                            break  # go to next obs_space_name in OBS_YAML
 
                         obsprep_space['window begin'] = self.window_begin
                         obsprep_space['window end'] = self.window_end
-                        ioda_config_file = obtype + '2ioda.yaml'
+                        ioda_config_file = obs_space_name + '2ioda.yaml'
                         obsprep_space['conversion config file'] = ioda_config_file
 
                         # set up the config file for conversion to IODA for bufr and
                         # netcdf files respectively
                         if obsprep_space['type'] == 'bufr':
+
                             # create a pre-filled template file for the bufr2ioda converter,
                             # which will be overwritten for each input cycle
                             bufrconv_config = {
@@ -164,12 +163,13 @@ def initialize(self):
                                 'DMPDIR': COMIN_OBS,
                                 'COM_OBS': COMIN_OBS,
                                 'OCEAN_BASIN_FILE': OCEAN_BASIN_FILE}
-                            bufr2iodapy = os.path.join(BUFR2IODA_PY_DIR, f'bufr2ioda_{obtype}.py')
+                            bufr2iodapy = os.path.join(BUFR2IODA_PY_DIR, f'bufr2ioda_{obs_space_name}.py')
                             obsprep_space['bufr2ioda converter'] = bufr2iodapy
-                            tmpl_filename = f"bufr2ioda_{obtype}.yaml"
+                            tmpl_filename = f"bufr2ioda_{obs_space_name}.yaml"
                             bufrconv_template = os.path.join(BUFR2IODA_TMPL_DIR, tmpl_filename)
                             input_files = []  # files to save to COM directory
                             bufrconv_files = []  # files needed to populate the IODA converter config
+
                             # for each cycle of the retrieved obs bufr files...
                             for input_file, cycle in fetched_files:
                                 cycletime = cycle[8:10]
@@ -178,19 +178,17 @@ def initialize(self):
                                 input_files.append(ioda_filename)
 
                             obsprep_space['bufrconv files'] = bufrconv_files
-
                             # set up config for concatenation
                             concat_config = {
                                 'provider': 'INSITUOBS',
                                 'window begin': obsprep_space['window begin'],
                                 'window end': obsprep_space['window end'],
-                                'variable': observer['obs space']['observed variables'][0],
+                                'variable': obs_space['observed variables'][0],
                                 'error ratio': obsprep_space['error ratio'],
                                 'input files': input_files,
                                 'output file': f"{RUN}.t{cycletime}z.{obs_space_name}.{cdatestr}.nc4"
                             }
-                            print('concat_config:', concat_config)
-                            concat_config_file = obtype + '_concat.yaml'
+                            concat_config_file = obs_space_name + '_concat.yaml'
 
                             obsprep_space['output file'] = concat_config['output file']
 
@@ -201,7 +199,7 @@ def initialize(self):
                                 save_as_yaml(concat_config, concat_config_file)
                             except Exception as e:
                                 logger.warning(f"An exeception {e} occured while trying to create BUFR2IODA config")
-                                logger.warning(f"obtype {obtype} will be skipped")
+                                logger.warning(f"obs_space_name {obs_space_name} will be skipped")
                                 break  # go to next observer in OBS_YAML
 
                             obsspaces_to_convert.append({"obs space": obsprep_space})
@@ -216,7 +214,7 @@ def initialize(self):
                             obsspaces_to_convert.append({"obs space": obsprep_space})
 
                         else:
-                            logger.warning(f"obs space {obtype} has bad type {obsprep_space['type']}, skipping")
+                            logger.warning(f"obs space {obs_space_name} has bad type {obsprep_space['type']}, skipping")
 
         except TypeError:
             logger.critical("Ill-formed OBS_YAML or OBSPREP_YAML file, exiting")
@@ -247,15 +245,15 @@ def run(self):
         for observation in obsspaces_to_convert['observations']:
 
             obs_space = observation['obs space']
-            obtype = obs_space['name']
-            logger.info(f"Trying to convert {obtype} to IODA")
+            obs_space_name = obs_space['name']
+            logger.info(f"Trying to convert {obs_space_name} to IODA")
             if obs_space["type"] == "nc":
                 process = Process(target=prep_ocean_obs_utils.run_netcdf_to_ioda, args=(obs_space,
                                                                                         self.task_config.OCNOBS2IODAEXEC))
             elif obs_space["type"] == "bufr":
                 process = Process(target=prep_ocean_obs_utils.run_bufr_to_ioda, args=(obs_space, self.task_config.OCNOBS2IODAEXEC))
             else:
-                logger.warning(f"Invalid observation format {obs_space['type']}, skipping obtype {obtype}")
+                logger.warning(f"Invalid observation format {obs_space['type']}, skipping obs_space_name {obs_space_name}")
                 continue
             process.start()
             processes.append((process, obs_space))