From da98e0a40863a5e4dcb71eedc34e71c0004601fc Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 27 Feb 2025 15:57:15 +0000 Subject: [PATCH 1/7] removed comments --- parm/soca/obs/obs_list.yaml.j2 | 43 ++++++++++++++++++++++++++++++++++ ush/soca/prep_ocean_obs.py | 34 ++++++++++----------------- 2 files changed, 56 insertions(+), 21 deletions(-) create mode 100644 parm/soca/obs/obs_list.yaml.j2 diff --git a/parm/soca/obs/obs_list.yaml.j2 b/parm/soca/obs/obs_list.yaml.j2 new file mode 100644 index 000000000..56b9c3423 --- /dev/null +++ b/parm/soca/obs/obs_list.yaml.j2 @@ -0,0 +1,43 @@ +observers: +# ADT +- adt_rads_all + +# SST +- sst_avhrr_ma_l3u +- sst_avhrr_mb_l3u +- sst_avhrr_mc_l3u +- sst_viirs_npp_l3u +- sst_viirs_n20_l3u +- sst_abi_g16_l3c +- sst_abi_g17_l3c +- sst_ahi_h08_l3c + +# Ice concentration +- icec_amsr2_north +- icec_amsr2_south +#- icec_viirs_n20_l2_north +#- icec_viirs_n20_l2_south +#- icec_amsu_mb_l2 +#- icec_atms_n20_l2 +#- icec_atms_n21_l2 +#- icec_atms_npp_l2 +#- icec_gmi_gpm_l2 +#- icec_ssmis_f17_l2 + +# in situ: monthly +#- insitu_profile_bathy +- insitu_profile_argo +#- insitu_profile_glider +#- insitu_profile_tesac +#- insitu_profile_tesac_salinity +#- insitu_profile_marinemammal +#- insitu_profile_xbtctd +#- insitu_surface_altkob +#- insitu_surface_trkob +#- insitu_surface_trkob_salinity + +# in situ: daily +#- insitu_profile_dbuoy +#- insitu_profile_dbuoyb +#- insitu_profile_mbuoy +#- insitu_profile_mbuoyb diff --git a/ush/soca/prep_ocean_obs.py b/ush/soca/prep_ocean_obs.py index e751964ab..31dbfff14 100644 --- a/ush/soca/prep_ocean_obs.py +++ b/ush/soca/prep_ocean_obs.py @@ -82,8 +82,8 @@ def initialize(self): except OSError: logger.warning("Could not copy RECCAP2_region_masks_all_v20221025.nc") - OBS_YAML = self.task_config['OBS_YAML'] - observer_config = YAMLFile(OBS_YAML) + OBS_YAML = os.path.join(self.task_config['PARMgfs'], 'gdas/soca/obs/obs_list.yaml.j2') + observers = parse_j2yaml(OBS_YAML, self.task_config)['observers'] OBSPREP_YAML = self.task_config['OBSPREP_YAML'] if os.path.exists(OBSPREP_YAML): @@ -108,23 +108,15 @@ def initialize(self): try: # go through the sources in OBS_YAML - for observer in observer_config['observers']: - try: - obs_space_name = observer['obs space']['name'] - logger.info(f"Trying to find observation {obs_space_name} in OBSPREP_YAML") - except KeyError: - logger.warning("Ill-formed observer yaml file, skipping") - continue + for observer in observers: # find match to the obs space from OBS_YAML in OBSPREP_YAML # this is awkward and unpythonic, so feel free to improve for obsprep_entry in obsprep_config['observations']: - obsprep_space = obsprep_entry['obs space'] + obsprep_space = obsprep_entry['obs space'] # the whole thing is needed later obsprep_space_name = obsprep_space['name'] - - if obsprep_space_name == obs_space_name: - obtype = obsprep_space_name # for brevity - logger.info(f"Observer {obtype} found in OBSPREP_YAML") + if obsprep_space_name == observer: + logger.info(f"Observer {observer} found in OBSPREP_YAML") try: obs_window_back = obsprep_space['window']['back'] @@ -145,12 +137,12 @@ def initialize(self): window_cdates) if not fetched_files: - logger.warning(f"No files found for obs source {obtype}, skipping") + logger.warning(f"No files found for obs source {observer}, skipping") break # go to next observer in OBS_YAML obsprep_space['window begin'] = self.window_begin obsprep_space['window end'] = self.window_end - ioda_config_file = obtype + '2ioda.yaml' + ioda_config_file = observer + '2ioda.yaml' obsprep_space['conversion config file'] = ioda_config_file # set up the config file for conversion to IODA for bufr and @@ -164,16 +156,16 @@ def initialize(self): 'DMPDIR': COMIN_OBS, 'COM_OBS': COMIN_OBS, 'OCEAN_BASIN_FILE': OCEAN_BASIN_FILE} - bufr2iodapy = os.path.join(BUFR2IODA_PY_DIR, f'bufr2ioda_{obtype}.py') + bufr2iodapy = os.path.join(BUFR2IODA_PY_DIR, f'bufr2ioda_{observer}.py') obsprep_space['bufr2ioda converter'] = bufr2iodapy - tmpl_filename = f"bufr2ioda_{obtype}.yaml" + tmpl_filename = f"bufr2ioda_{observer}.yaml" bufrconv_template = os.path.join(BUFR2IODA_TMPL_DIR, tmpl_filename) output_files = [] # files to save to COM directory bufrconv_files = [] # files needed to populate the IODA converter config # for each cycle of the retrieved obs bufr files... for input_file, cycle in fetched_files: cycletime = cycle[8:10] - ioda_filename = f"{RUN}.t{cycletime}z.{obs_space_name}.{cycle}.nc4" + ioda_filename = f"{RUN}.t{cycletime}z.{observer}.{cycle}.nc4" output_files.append(ioda_filename) bufrconv_files.append((cycle, input_file, ioda_filename)) @@ -194,14 +186,14 @@ def initialize(self): elif obsprep_space['type'] == 'nc': obsprep_space['input files'] = [f[0] for f in fetched_files] - ioda_filename = f"{RUN}.t{cyc:02d}z.{obs_space_name}.{cdatestr}.nc4" + ioda_filename = f"{RUN}.t{cyc:02d}z.{observer}.{cdatestr}.nc4" obsprep_space['output file'] = [ioda_filename] save_as_yaml(obsprep_space, ioda_config_file) obsspaces_to_convert.append({"obs space": obsprep_space}) else: - logger.warning(f"obs space {obtype} has bad type {obsprep_space['type']}, skipping") + logger.warning(f"obs space {observer} has bad type {obsprep_space['type']}, skipping") except TypeError: logger.critical("Ill-formed OBS_YAML or OBSPREP_YAML file, exiting") From 825a291a0b50abd158d6d60ed75708a6db52c494 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 6 Mar 2025 20:18:45 +0000 Subject: [PATCH 2/7] mods to obs list files --- parm/soca/obs/obs_list.yaml.j2 | 5 +++-- parm/soca/obs/obs_list_base_yaml.j2 | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 parm/soca/obs/obs_list_base_yaml.j2 diff --git a/parm/soca/obs/obs_list.yaml.j2 b/parm/soca/obs/obs_list.yaml.j2 index 56b9c3423..0598eb625 100644 --- a/parm/soca/obs/obs_list.yaml.j2 +++ b/parm/soca/obs/obs_list.yaml.j2 @@ -1,4 +1,4 @@ -observers: +observations: # ADT - adt_rads_all @@ -26,7 +26,8 @@ observers: # in situ: monthly #- insitu_profile_bathy -- insitu_profile_argo +#- insitu_temp_profile_argo +#- insitu_salt_profile_argo #- insitu_profile_glider #- insitu_profile_tesac #- insitu_profile_tesac_salinity diff --git a/parm/soca/obs/obs_list_base_yaml.j2 b/parm/soca/obs/obs_list_base_yaml.j2 new file mode 100644 index 000000000..8ceec6d6d --- /dev/null +++ b/parm/soca/obs/obs_list_base_yaml.j2 @@ -0,0 +1,15 @@ +#cost function: + +# observations: +# obs perturbations: {{ obs_perturbations | default(false, true) }} + observers: +{% for observation_from_jcb in observations %} +{% if use_observer(observation_from_jcb) %} +{% filter indent(width=4) %} +{% set file_path = app_path_observations + '/' + observation_from_jcb + '.yaml.j2' %} +{% include file_path %} +{% endfilter %} +{% endif %} +{% endfor %} + + From 3b3fc4d92688cf440b030c613f8afee87e9b5c80 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 6 Mar 2025 20:21:08 +0000 Subject: [PATCH 3/7] mods to script --- ush/soca/prep_ocean_obs.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ush/soca/prep_ocean_obs.py b/ush/soca/prep_ocean_obs.py index bb42bdea9..ec3fc9ca5 100644 --- a/ush/soca/prep_ocean_obs.py +++ b/ush/soca/prep_ocean_obs.py @@ -82,8 +82,13 @@ def initialize(self): except OSError: logger.warning("Could not copy RECCAP2_region_masks_all_v20221025.nc") + self.task_config.app_path_observations = os.path.join(self.task_config['PARMgfs'], 'gdas/jcb-gdas/observations/marine') OBS_YAML = os.path.join(self.task_config['PARMgfs'], 'gdas/soca/obs/obs_list.yaml.j2') - observers = parse_j2yaml(OBS_YAML, self.task_config)['observers'] + self.task_config.observations = parse_j2yaml(OBS_YAML, self.task_config)['observations'] + + obsconfigfile = os.path.join(self.task_config['PARMgfs'], 'gdas/soca/obs/obs_list_base_yaml.j2') + obsconfig = parse_j2yaml(obsconfigfile, self.task_config) + print('obsconfig:',obsconfig) OBSPREP_YAML = self.task_config['OBSPREP_YAML'] if os.path.exists(OBSPREP_YAML): @@ -108,7 +113,7 @@ def initialize(self): try: # go through the sources in OBS_YAML - for observer in observers: + for observer in self.task_config.observations: # find match to the obs space from OBS_YAML in OBSPREP_YAML # this is awkward and unpythonic, so feel free to improve @@ -148,6 +153,7 @@ def initialize(self): # set up the config file for conversion to IODA for bufr and # netcdf files respectively if obsprep_space['type'] == 'bufr': + # create a pre-filled template file for the bufr2ioda converter, # which will be overwritten for each input cycle bufrconv_config = { @@ -162,13 +168,12 @@ def initialize(self): bufrconv_template = os.path.join(BUFR2IODA_TMPL_DIR, tmpl_filename) input_files = [] # files to save to COM directory bufrconv_files = [] # files needed to populate the IODA converter config + # for each cycle of the retrieved obs bufr files... for input_file, cycle in fetched_files: cycletime = cycle[8:10] ioda_filename = f"{RUN}.t{cycletime}z.{observer}.{cycle}.nc4" - output_files.append(ioda_filename) bufrconv_files.append((cycle, input_file, ioda_filename)) - #??? input_files.append(ioda_filename) obsprep_space['bufrconv files'] = bufrconv_files From 111bf85e96d3c423876bad1c3e085b84521dea99 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 6 Mar 2025 21:41:34 +0000 Subject: [PATCH 4/7] lets gooooo --- ush/soca/prep_ocean_obs.py | 41 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/ush/soca/prep_ocean_obs.py b/ush/soca/prep_ocean_obs.py index ec3fc9ca5..d01f7972f 100644 --- a/ush/soca/prep_ocean_obs.py +++ b/ush/soca/prep_ocean_obs.py @@ -87,8 +87,7 @@ def initialize(self): self.task_config.observations = parse_j2yaml(OBS_YAML, self.task_config)['observations'] obsconfigfile = os.path.join(self.task_config['PARMgfs'], 'gdas/soca/obs/obs_list_base_yaml.j2') - obsconfig = parse_j2yaml(obsconfigfile, self.task_config) - print('obsconfig:',obsconfig) + obsconfig = parse_j2yaml(obsconfigfile, self.task_config)['observers'] OBSPREP_YAML = self.task_config['OBSPREP_YAML'] if os.path.exists(OBSPREP_YAML): @@ -113,15 +112,17 @@ def initialize(self): try: # go through the sources in OBS_YAML - for observer in self.task_config.observations: + for observation in obsconfig: + obs_space = observation['obs space'] + obs_space_name = obs_space['name'] # find match to the obs space from OBS_YAML in OBSPREP_YAML # this is awkward and unpythonic, so feel free to improve for obsprep_entry in obsprep_config['observations']: obsprep_space = obsprep_entry['obs space'] # the whole thing is needed later obsprep_space_name = obsprep_space['name'] - if obsprep_space_name == observer: - logger.info(f"Observer {observer} found in OBSPREP_YAML") + if obsprep_space_name == obs_space_name: + logger.info(f"Observer {obs_space_name} found in OBSPREP_YAML") try: obs_window_back = obsprep_space['window']['back'] @@ -142,12 +143,12 @@ def initialize(self): window_cdates) if not fetched_files: - logger.warning(f"No files found for obs source {observer}, skipping") - break # go to next observer in OBS_YAML + logger.warning(f"No files found for obs source {obs_space_name}, skipping") + break # go to next obs_space_name in OBS_YAML obsprep_space['window begin'] = self.window_begin obsprep_space['window end'] = self.window_end - ioda_config_file = observer + '2ioda.yaml' + ioda_config_file = obs_space_name + '2ioda.yaml' obsprep_space['conversion config file'] = ioda_config_file # set up the config file for conversion to IODA for bufr and @@ -162,9 +163,9 @@ def initialize(self): 'DMPDIR': COMIN_OBS, 'COM_OBS': COMIN_OBS, 'OCEAN_BASIN_FILE': OCEAN_BASIN_FILE} - bufr2iodapy = os.path.join(BUFR2IODA_PY_DIR, f'bufr2ioda_{observer}.py') + bufr2iodapy = os.path.join(BUFR2IODA_PY_DIR, f'bufr2ioda_{obs_space_name}.py') obsprep_space['bufr2ioda converter'] = bufr2iodapy - tmpl_filename = f"bufr2ioda_{observer}.yaml" + tmpl_filename = f"bufr2ioda_{obs_space_name}.yaml" bufrconv_template = os.path.join(BUFR2IODA_TMPL_DIR, tmpl_filename) input_files = [] # files to save to COM directory bufrconv_files = [] # files needed to populate the IODA converter config @@ -172,24 +173,24 @@ def initialize(self): # for each cycle of the retrieved obs bufr files... for input_file, cycle in fetched_files: cycletime = cycle[8:10] - ioda_filename = f"{RUN}.t{cycletime}z.{observer}.{cycle}.nc4" + ioda_filename = f"{RUN}.t{cycletime}z.{obs_space_name}.{cycle}.nc4" bufrconv_files.append((cycle, input_file, ioda_filename)) input_files.append(ioda_filename) obsprep_space['bufrconv files'] = bufrconv_files - + print('obs_space_name: ',obs_space_name) # set up config for concatenation concat_config = { 'provider': 'INSITUOBS', 'window begin': obsprep_space['window begin'], 'window end': obsprep_space['window end'], - 'variable': observer['obs space']['observed variables'][0], + 'variable': obs_space['observed variables'][0], 'error ratio': obsprep_space['error ratio'], 'input files': input_files, 'output file': f"{RUN}.t{cycletime}z.{obs_space_name}.{cdatestr}.nc4" } print('concat_config:', concat_config) - concat_config_file = obtype + '_concat.yaml' + concat_config_file = obs_space_name + '_concat.yaml' obsprep_space['output file'] = concat_config['output file'] @@ -200,7 +201,7 @@ def initialize(self): save_as_yaml(concat_config, concat_config_file) except Exception as e: logger.warning(f"An exeception {e} occured while trying to create BUFR2IODA config") - logger.warning(f"obtype {obtype} will be skipped") + logger.warning(f"obs_space_name {obs_space_name} will be skipped") break # go to next observer in OBS_YAML obsspaces_to_convert.append({"obs space": obsprep_space}) @@ -208,14 +209,14 @@ def initialize(self): elif obsprep_space['type'] == 'nc': obsprep_space['input files'] = [f[0] for f in fetched_files] - ioda_filename = f"{RUN}.t{cyc:02d}z.{observer}.{cdatestr}.nc4" + ioda_filename = f"{RUN}.t{cyc:02d}z.{obs_space_name}.{cdatestr}.nc4" obsprep_space['output file'] = ioda_filename save_as_yaml(obsprep_space, ioda_config_file) obsspaces_to_convert.append({"obs space": obsprep_space}) else: - logger.warning(f"obs space {observer} has bad type {obsprep_space['type']}, skipping") + logger.warning(f"obs space {obs_space_name} has bad type {obsprep_space['type']}, skipping") except TypeError: logger.critical("Ill-formed OBS_YAML or OBSPREP_YAML file, exiting") @@ -246,15 +247,15 @@ def run(self): for observation in obsspaces_to_convert['observations']: obs_space = observation['obs space'] - obtype = obs_space['name'] - logger.info(f"Trying to convert {obtype} to IODA") + obs_space_name = obs_space['name'] + logger.info(f"Trying to convert {obs_space_name} to IODA") if obs_space["type"] == "nc": process = Process(target=prep_ocean_obs_utils.run_netcdf_to_ioda, args=(obs_space, self.task_config.OCNOBS2IODAEXEC)) elif obs_space["type"] == "bufr": process = Process(target=prep_ocean_obs_utils.run_bufr_to_ioda, args=(obs_space, self.task_config.OCNOBS2IODAEXEC)) else: - logger.warning(f"Invalid observation format {obs_space['type']}, skipping obtype {obtype}") + logger.warning(f"Invalid observation format {obs_space['type']}, skipping obs_space_name {obs_space_name}") continue process.start() processes.append((process, obs_space)) From e05164a40d4f3dd96a23907008ebda2edb8acc03 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 6 Mar 2025 21:55:15 +0000 Subject: [PATCH 5/7] style, removed print statement --- ush/soca/prep_ocean_obs.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ush/soca/prep_ocean_obs.py b/ush/soca/prep_ocean_obs.py index d01f7972f..6b0379576 100644 --- a/ush/soca/prep_ocean_obs.py +++ b/ush/soca/prep_ocean_obs.py @@ -84,7 +84,7 @@ def initialize(self): self.task_config.app_path_observations = os.path.join(self.task_config['PARMgfs'], 'gdas/jcb-gdas/observations/marine') OBS_YAML = os.path.join(self.task_config['PARMgfs'], 'gdas/soca/obs/obs_list.yaml.j2') - self.task_config.observations = parse_j2yaml(OBS_YAML, self.task_config)['observations'] + self.task_config.observations = parse_j2yaml(OBS_YAML, self.task_config)['observations'] obsconfigfile = os.path.join(self.task_config['PARMgfs'], 'gdas/soca/obs/obs_list_base_yaml.j2') obsconfig = parse_j2yaml(obsconfigfile, self.task_config)['observers'] @@ -119,7 +119,7 @@ def initialize(self): # find match to the obs space from OBS_YAML in OBSPREP_YAML # this is awkward and unpythonic, so feel free to improve for obsprep_entry in obsprep_config['observations']: - obsprep_space = obsprep_entry['obs space'] # the whole thing is needed later + obsprep_space = obsprep_entry['obs space'] # the whole thing is needed later obsprep_space_name = obsprep_space['name'] if obsprep_space_name == obs_space_name: logger.info(f"Observer {obs_space_name} found in OBSPREP_YAML") @@ -178,7 +178,6 @@ def initialize(self): input_files.append(ioda_filename) obsprep_space['bufrconv files'] = bufrconv_files - print('obs_space_name: ',obs_space_name) # set up config for concatenation concat_config = { 'provider': 'INSITUOBS', @@ -189,7 +188,6 @@ def initialize(self): 'input files': input_files, 'output file': f"{RUN}.t{cycletime}z.{obs_space_name}.{cdatestr}.nc4" } - print('concat_config:', concat_config) concat_config_file = obs_space_name + '_concat.yaml' obsprep_space['output file'] = concat_config['output file'] From e512d1ac2257aeb833bbb1715f90ba7e105c4846 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Fri, 7 Mar 2025 14:31:21 +0000 Subject: [PATCH 6/7] corrected comments --- parm/soca/obs/obs_list_base_yaml.j2 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/parm/soca/obs/obs_list_base_yaml.j2 b/parm/soca/obs/obs_list_base_yaml.j2 index 8ceec6d6d..dc773fa20 100644 --- a/parm/soca/obs/obs_list_base_yaml.j2 +++ b/parm/soca/obs/obs_list_base_yaml.j2 @@ -1,7 +1,5 @@ -#cost function: +# used with obs list to load obs configs -# observations: -# obs perturbations: {{ obs_perturbations | default(false, true) }} observers: {% for observation_from_jcb in observations %} {% if use_observer(observation_from_jcb) %} From 334e64edabca132703cd7e04f8fe67e8a0a81776 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Mon, 10 Mar 2025 16:29:25 +0000 Subject: [PATCH 7/7] added missing obs types --- parm/soca/obs/obs_list.yaml.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/parm/soca/obs/obs_list.yaml.j2 b/parm/soca/obs/obs_list.yaml.j2 index 0598eb625..7ffc869b8 100644 --- a/parm/soca/obs/obs_list.yaml.j2 +++ b/parm/soca/obs/obs_list.yaml.j2 @@ -2,6 +2,10 @@ observations: # ADT - adt_rads_all +#SSS +- sss_smap_l2 +- sss_smos_l2 + # SST - sst_avhrr_ma_l3u - sst_avhrr_mb_l3u @@ -42,3 +46,5 @@ observations: #- insitu_profile_dbuoyb #- insitu_profile_mbuoy #- insitu_profile_mbuoyb +- insitu_profile_tropical +- insitu_surface_drifter