diff --git a/parm/soca/obs/obs_list.yaml.j2 b/parm/soca/obs/obs_list.yaml.j2 new file mode 100644 index 000000000..7ffc869b8 --- /dev/null +++ b/parm/soca/obs/obs_list.yaml.j2 @@ -0,0 +1,50 @@ +observations: +# ADT +- adt_rads_all + +#SSS +- sss_smap_l2 +- sss_smos_l2 + +# SST +- sst_avhrr_ma_l3u +- sst_avhrr_mb_l3u +- sst_avhrr_mc_l3u +- sst_viirs_npp_l3u +- sst_viirs_n20_l3u +- sst_abi_g16_l3c +- sst_abi_g17_l3c +- sst_ahi_h08_l3c + +# Ice concentration +- icec_amsr2_north +- icec_amsr2_south +#- icec_viirs_n20_l2_north +#- icec_viirs_n20_l2_south +#- icec_amsu_mb_l2 +#- icec_atms_n20_l2 +#- icec_atms_n21_l2 +#- icec_atms_npp_l2 +#- icec_gmi_gpm_l2 +#- icec_ssmis_f17_l2 + +# in situ: monthly +#- insitu_profile_bathy +#- insitu_temp_profile_argo +#- insitu_salt_profile_argo +#- insitu_profile_glider +#- insitu_profile_tesac +#- insitu_profile_tesac_salinity +#- insitu_profile_marinemammal +#- insitu_profile_xbtctd +#- insitu_surface_altkob +#- insitu_surface_trkob +#- insitu_surface_trkob_salinity + +# in situ: daily +#- insitu_profile_dbuoy +#- insitu_profile_dbuoyb +#- insitu_profile_mbuoy +#- insitu_profile_mbuoyb +- insitu_profile_tropical +- insitu_surface_drifter diff --git a/parm/soca/obs/obs_list_base_yaml.j2 b/parm/soca/obs/obs_list_base_yaml.j2 new file mode 100644 index 000000000..dc773fa20 --- /dev/null +++ b/parm/soca/obs/obs_list_base_yaml.j2 @@ -0,0 +1,13 @@ +# used with obs list to load obs configs + + observers: +{% for observation_from_jcb in observations %} +{% if use_observer(observation_from_jcb) %} +{% filter indent(width=4) %} +{% set file_path = app_path_observations + '/' + observation_from_jcb + '.yaml.j2' %} +{% include file_path %} +{% endfilter %} +{% endif %} +{% endfor %} + + diff --git a/ush/soca/prep_ocean_obs.py b/ush/soca/prep_ocean_obs.py index 71f62416f..6b0379576 100644 --- a/ush/soca/prep_ocean_obs.py +++ b/ush/soca/prep_ocean_obs.py @@ -82,8 +82,12 @@ def initialize(self): except OSError: logger.warning("Could not copy RECCAP2_region_masks_all_v20221025.nc") - OBS_YAML = self.task_config['OBS_YAML'] - observer_config = YAMLFile(OBS_YAML) + self.task_config.app_path_observations = os.path.join(self.task_config['PARMgfs'], 'gdas/jcb-gdas/observations/marine') + OBS_YAML = os.path.join(self.task_config['PARMgfs'], 'gdas/soca/obs/obs_list.yaml.j2') + self.task_config.observations = parse_j2yaml(OBS_YAML, self.task_config)['observations'] + + obsconfigfile = os.path.join(self.task_config['PARMgfs'], 'gdas/soca/obs/obs_list_base_yaml.j2') + obsconfig = parse_j2yaml(obsconfigfile, self.task_config)['observers'] OBSPREP_YAML = self.task_config['OBSPREP_YAML'] if os.path.exists(OBSPREP_YAML): @@ -108,23 +112,17 @@ def initialize(self): try: # go through the sources in OBS_YAML - for observer in observer_config['observers']: - try: - obs_space_name = observer['obs space']['name'] - logger.info(f"Trying to find observation {obs_space_name} in OBSPREP_YAML") - except KeyError: - logger.warning("Ill-formed observer yaml file, skipping") - continue + for observation in obsconfig: + obs_space = observation['obs space'] + obs_space_name = obs_space['name'] # find match to the obs space from OBS_YAML in OBSPREP_YAML # this is awkward and unpythonic, so feel free to improve for obsprep_entry in obsprep_config['observations']: - obsprep_space = obsprep_entry['obs space'] + obsprep_space = obsprep_entry['obs space'] # the whole thing is needed later obsprep_space_name = obsprep_space['name'] - if obsprep_space_name == obs_space_name: - obtype = obsprep_space_name # for brevity - logger.info(f"Observer {obtype} found in OBSPREP_YAML") + logger.info(f"Observer {obs_space_name} found in OBSPREP_YAML") try: obs_window_back = obsprep_space['window']['back'] @@ -145,17 +143,18 @@ def initialize(self): window_cdates) if not fetched_files: - logger.warning(f"No files found for obs source {obtype}, skipping") - break # go to next observer in OBS_YAML + logger.warning(f"No files found for obs source {obs_space_name}, skipping") + break # go to next obs_space_name in OBS_YAML obsprep_space['window begin'] = self.window_begin obsprep_space['window end'] = self.window_end - ioda_config_file = obtype + '2ioda.yaml' + ioda_config_file = obs_space_name + '2ioda.yaml' obsprep_space['conversion config file'] = ioda_config_file # set up the config file for conversion to IODA for bufr and # netcdf files respectively if obsprep_space['type'] == 'bufr': + # create a pre-filled template file for the bufr2ioda converter, # which will be overwritten for each input cycle bufrconv_config = { @@ -164,12 +163,13 @@ def initialize(self): 'DMPDIR': COMIN_OBS, 'COM_OBS': COMIN_OBS, 'OCEAN_BASIN_FILE': OCEAN_BASIN_FILE} - bufr2iodapy = os.path.join(BUFR2IODA_PY_DIR, f'bufr2ioda_{obtype}.py') + bufr2iodapy = os.path.join(BUFR2IODA_PY_DIR, f'bufr2ioda_{obs_space_name}.py') obsprep_space['bufr2ioda converter'] = bufr2iodapy - tmpl_filename = f"bufr2ioda_{obtype}.yaml" + tmpl_filename = f"bufr2ioda_{obs_space_name}.yaml" bufrconv_template = os.path.join(BUFR2IODA_TMPL_DIR, tmpl_filename) input_files = [] # files to save to COM directory bufrconv_files = [] # files needed to populate the IODA converter config + # for each cycle of the retrieved obs bufr files... for input_file, cycle in fetched_files: cycletime = cycle[8:10] @@ -178,19 +178,17 @@ def initialize(self): input_files.append(ioda_filename) obsprep_space['bufrconv files'] = bufrconv_files - # set up config for concatenation concat_config = { 'provider': 'INSITUOBS', 'window begin': obsprep_space['window begin'], 'window end': obsprep_space['window end'], - 'variable': observer['obs space']['observed variables'][0], + 'variable': obs_space['observed variables'][0], 'error ratio': obsprep_space['error ratio'], 'input files': input_files, 'output file': f"{RUN}.t{cycletime}z.{obs_space_name}.{cdatestr}.nc4" } - print('concat_config:', concat_config) - concat_config_file = obtype + '_concat.yaml' + concat_config_file = obs_space_name + '_concat.yaml' obsprep_space['output file'] = concat_config['output file'] @@ -201,7 +199,7 @@ def initialize(self): save_as_yaml(concat_config, concat_config_file) except Exception as e: logger.warning(f"An exeception {e} occured while trying to create BUFR2IODA config") - logger.warning(f"obtype {obtype} will be skipped") + logger.warning(f"obs_space_name {obs_space_name} will be skipped") break # go to next observer in OBS_YAML obsspaces_to_convert.append({"obs space": obsprep_space}) @@ -216,7 +214,7 @@ def initialize(self): obsspaces_to_convert.append({"obs space": obsprep_space}) else: - logger.warning(f"obs space {obtype} has bad type {obsprep_space['type']}, skipping") + logger.warning(f"obs space {obs_space_name} has bad type {obsprep_space['type']}, skipping") except TypeError: logger.critical("Ill-formed OBS_YAML or OBSPREP_YAML file, exiting") @@ -247,15 +245,15 @@ def run(self): for observation in obsspaces_to_convert['observations']: obs_space = observation['obs space'] - obtype = obs_space['name'] - logger.info(f"Trying to convert {obtype} to IODA") + obs_space_name = obs_space['name'] + logger.info(f"Trying to convert {obs_space_name} to IODA") if obs_space["type"] == "nc": process = Process(target=prep_ocean_obs_utils.run_netcdf_to_ioda, args=(obs_space, self.task_config.OCNOBS2IODAEXEC)) elif obs_space["type"] == "bufr": process = Process(target=prep_ocean_obs_utils.run_bufr_to_ioda, args=(obs_space, self.task_config.OCNOBS2IODAEXEC)) else: - logger.warning(f"Invalid observation format {obs_space['type']}, skipping obtype {obtype}") + logger.warning(f"Invalid observation format {obs_space['type']}, skipping obs_space_name {obs_space_name}") continue process.start() processes.append((process, obs_space))