diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_salt_profile_argo.yaml b/parm/ioda/bufr2ioda/bufr2ioda_insitu_salt_profile_argo.yaml new file mode 100644 index 000000000..7dd28f21c --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_salt_profile_argo.yaml @@ -0,0 +1,11 @@ +cycle_datetime: '{{ current_cycle | to_YMDH }}' +cycle_type: '{{ RUN }}' +data_description: 6-hrly in situ ARGO profiles +data_format: subpfl +data_provider: U.S. NOAA +data_type: argo +dump_directory: '{{ DMPDIR }}' +ioda_directory: '{{ COM_OBS }}' +source: NCEP data tank +subsets: SUBPFL +ocean_basin: '{{ OCEAN_BASIN_FILE }}' diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_temp_profile_argo.yaml b/parm/ioda/bufr2ioda/bufr2ioda_insitu_temp_profile_argo.yaml new file mode 100644 index 000000000..7dd28f21c --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_temp_profile_argo.yaml @@ -0,0 +1,11 @@ +cycle_datetime: '{{ current_cycle | to_YMDH }}' +cycle_type: '{{ RUN }}' +data_description: 6-hrly in situ ARGO profiles +data_format: subpfl +data_provider: U.S. NOAA +data_type: argo +dump_directory: '{{ DMPDIR }}' +ioda_directory: '{{ COM_OBS }}' +source: NCEP data tank +subsets: SUBPFL +ocean_basin: '{{ OCEAN_BASIN_FILE }}' diff --git a/parm/soca/obs/config/insitu_salt_profile_argo.yaml b/parm/soca/obs/config/insitu_salt_profile_argo.yaml new file mode 100644 index 000000000..0f4247222 --- /dev/null +++ b/parm/soca/obs/config/insitu_salt_profile_argo.yaml @@ -0,0 +1,179 @@ +obs space: + name: insitu_salt_profile_argo + obsdatain: + engine: + type: H5File + obsfile: ${DATA}/obs/${OPREFIX}insitu_salt_profile_argo.${PDY}${cyc}.nc4 + obsgrouping: + group variables: [latitude, longitude, dateTime] + sort variable: depth + sort group: MetaData + sort order: ascending + obsdataout: + engine: + type: H5File + obsfile: ${DATA}/diags/insitu_salt_profile_argo.${PDY}${cyc}.nc4 + simulated variables: [salinity] + observed variables: [salinity] + io pool: + max pool size: 1 +obs operator: + name: VertInterp + observation alias file: ./obsop_name_map.yaml + vertical coordinate: sea_water_depth + observation vertical coordinate: depth + interpolation method: linear +obs error: + covariance model: diagonal + +#------------------------------------------------------------------------------- +# START OF OBS FILTERS (work done by Kriti Bhargava) +# The QC filters used here are based on the document by IODE that can be found at +# https://cdn.ioos.noaa.gov/media/2017/12/recommendations_in_situ_data_real_time_qc.pdf +#------------------------------------------------------------------------------- +obs filters: + + # land check + - filter: Domain Check + where: + - variable: {name: GeoVaLs/sea_area_fraction} + minvalue: 0.5 + + +#------------------------------------------------------------------------------- +## Filters for S: +#------------------------------------------------------------------------------- + #----------------------------------------------------------------------------- + ### Global range test + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 2.0 + maxvalue: 41.0 + + #----------------------------------------------------------------------------- + ### Regional range test + #----------------------------------------------------------------------------- + #### Red Sea + #----------------------------------------------------------------------------- + #### + #### the document linked here describes Red sea as the are between 10N, 40E; + #### 20N, 50E; 30N, 30E; 10N, 40E. But that would also include Gulf of Aden. + #### A more reasonable choice seemed to be a box that extends from 10 N to + #### 30 N and 30 E to 45 East . + + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 2.0 + maxvalue: 41.0 + where: + - variable: + name: MetaData/latitude + minvalue: 10 + maxvalue: 30 + - variable: + name: MetaData/longitude + minvalue: 30 + maxvalue: 45 + + #### Mediterranean Sea + #----------------------------------------------------------------------------- + ##### Area 1/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 2.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 30 + maxvalue: 40 + - variable: + name: MetaData/longitude + minvalue: -6 + maxvalue: 40 + ##### Area 2/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 2.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 40 + maxvalue: 41.5 + - variable: + name: MetaData/longitude + minvalue: 20 + maxvalue: 30 + ##### Area 3/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 2.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 40 + maxvalue: 46 + - variable: + name: MetaData/longitude + minvalue: 0 + maxvalue: 20 + + + #### Northwestern shelves + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 0.0 + maxvalue: 37.0 + where: + - variable: + name: MetaData/latitude + minvalue: 50 + maxvalue: 60 + - variable: + name: MetaData/longitude + minvalue: -20 + maxvalue: 10 + + #### Southwestern shelves + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 0.0 + maxvalue: 38 + where: + - variable: + name: MetaData/latitude + minvalue: 25 + maxvalue: 50 + - variable: + name: MetaData/longitude + minvalue: -30 + maxvalue: 0 + + #### Arctic Sea + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 2.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 60 + + - filter: Background Check + filter variables: [{name: salinity}] + threshold: 5.0 + absolute threshold: 5.0 + +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/config/insitu_temp_profile_argo.yaml b/parm/soca/obs/config/insitu_temp_profile_argo.yaml new file mode 100644 index 000000000..76f5503a6 --- /dev/null +++ b/parm/soca/obs/config/insitu_temp_profile_argo.yaml @@ -0,0 +1,173 @@ +obs space: + name: insitu_temp_profile_argo + obsdatain: + engine: + type: H5File + obsfile: ${DATA}/obs/${OPREFIX}insitu_temp_profile_argo.${PDY}${cyc}.nc4 + obsgrouping: + group variables: [latitude, longitude, dateTime] + sort variable: depth + sort group: MetaData + sort order: ascending + obsdataout: + engine: + type: H5File + obsfile: ${DATA}/diags/insitu_temp_profile_argo.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature] + observed variables: [waterTemperature] + io pool: + max pool size: 1 +obs operator: + name: InsituTemperature +obs error: + covariance model: diagonal + +#------------------------------------------------------------------------------- +# START OF OBS FILTERS (work done by Kriti Bhargava) +# The QC filters used here are based on the document by IODE that can be found at +# https://cdn.ioos.noaa.gov/media/2017/12/recommendations_in_situ_data_real_time_qc.pdf +#------------------------------------------------------------------------------- +obs filters: + + # land check + - filter: Domain Check + where: + - variable: {name: GeoVaLs/sea_area_fraction} + minvalue: 0.5 + +#------------------------------------------------------------------------------- +## Filters for T: +#------------------------------------------------------------------------------- + #------------------------------------------------------------------------------- + ### Global range test + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: -2.5 + maxvalue: 40.0 + + #----------------------------------------------------------------------------- + ### Regional range tests + #----------------------------------------------------------------------------- + + #### Red Sea + #----------------------------------------------------------------------------- + #### + #### the document linked here describes Red sea as the are between 10N, 40E; + #### 20N, 50E; 30N, 30E; 10N, 40E. But that would also include Gulf of Aden. + #### A more reasonable choice seemed to be a box that extends from 10 N to + #### 30 N and 30 E to 45 East . + + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: 21.7 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 10 + maxvalue: 30 + - variable: + name: MetaData/longitude + minvalue: 30 + maxvalue: 45 + + #### Mediterranean Sea + #----------------------------------------------------------------------------- + ##### Area 1/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: 10.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 30 + maxvalue: 40 + - variable: + name: MetaData/longitude + minvalue: -6 + maxvalue: 40 + ##### Area 2/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: 10.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 40 + maxvalue: 41.5 + - variable: + name: MetaData/longitude + minvalue: 20 + maxvalue: 30 + ##### Area 3/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: 10.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 40 + maxvalue: 46 + - variable: + name: MetaData/longitude + minvalue: 0 + maxvalue: 20 + + #### Northwestern shelves + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: -2.0 + maxvalue: 24.0 + where: + - variable: + name: MetaData/latitude + minvalue: 50 + maxvalue: 60 + - variable: + name: MetaData/longitude + minvalue: -20 + maxvalue: 10 + #### Southwestern shelves + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: -2.0 + maxvalue: 30 + where: + - variable: + name: MetaData/latitude + minvalue: 25 + maxvalue: 50 + - variable: + name: MetaData/longitude + minvalue: -30 + maxvalue: 0 + + #### Arctic Sea + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: -1.92 + maxvalue: 25.0 + where: + - variable: + name: MetaData/latitude + minvalue: 60 + + - filter: Background Check + filter variables: [{name: waterTemperature}] + threshold: 5.0 + absolute threshold: 5.0 + +obs localizations: +- localization method: Rossby + base value: 100.0e3 + rossby mult: 1.0 + min grid mult: 2.0 + min value: 200.0e3 + max value: 900.0e3 diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml index c11dc1ace..e1d2dde45 100644 --- a/parm/soca/obs/obs_list.yaml +++ b/parm/soca/obs/obs_list.yaml @@ -26,7 +26,9 @@ observers: # in situ: monthly #- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_bathy.yaml -- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_argo.yaml +#- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_argo.yaml +#- !INC ${MARINE_OBS_YAML_DIR}/insitu_temp_profile_argo.yaml +#- !INC ${MARINE_OBS_YAML_DIR}/insitu_salt_profile_argo.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_glider.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_tesac.yaml #- !INC ${MARINE_OBS_YAML_DIR}/insitu_profile_tesac_salinity.yaml diff --git a/parm/soca/obsprep/obsprep_config.yaml b/parm/soca/obsprep/obsprep_config.yaml index 02f4edb5d..576910e4d 100644 --- a/parm/soca/obsprep/obsprep_config.yaml +++ b/parm/soca/obsprep/obsprep_config.yaml @@ -240,10 +240,22 @@ observations: dmpdir regex: 'gdas.*.bathy.*.bufr_d' - obs space: - name: insitu_profile_argo + name: insitu_temp_profile_argo provider: GTS dmpdir subdir: atmos type: bufr + error ratio: 0.4 + window: + back: 4 + forward: 4 + dmpdir regex: 'gdas.*.subpfl.*.bufr_d' + +- obs space: + name: insitu_salt_profile_argo + provider: GTS + dmpdir subdir: atmos + type: bufr + error ratio: 0.4 window: back: 4 forward: 4 diff --git a/ush/ioda/bufr2ioda/marine/b2i/bufr2ioda_insitu_salt_profile_argo.py b/ush/ioda/bufr2ioda/marine/b2i/bufr2ioda_insitu_salt_profile_argo.py new file mode 100755 index 000000000..f8aa99aeb --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/b2i/bufr2ioda_insitu_salt_profile_argo.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import sys +from b2iconverter.util import parse_arguments +from b2iconverter.bufr2ioda_config import Bufr2iodaConfig +from b2iconverter.bufr2ioda_converter import Bufr2ioda_Converter +from argo_ioda_variables import ArgoIODAVariables + + +platform_description = 'ARGO profiles from subpfl: temperature and salinity' + + +class ArgoConfig(Bufr2iodaConfig): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_salt_profile_argo.{self.cycle_datetime}.nc4" + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + + bufr2ioda_config = ArgoConfig( + script_name, + config_file, + platform_description) + + ioda_vars = ArgoIODAVariables() + ioda_vars.set_temperature_var_name("waterTemperature") + ioda_vars.set_temperature_error(0.02) + ioda_vars.set_salinity_var_name("salinity") + ioda_vars.set_salinity_error(0.01) + + argo = Bufr2ioda_Converter(bufr2ioda_config, ioda_vars, log_file) + + argo.run() + + if test_file: + result = argo.test(test_file) + sys.exit(result) diff --git a/ush/ioda/bufr2ioda/marine/b2i/bufr2ioda_insitu_temp_profile_argo.py b/ush/ioda/bufr2ioda/marine/b2i/bufr2ioda_insitu_temp_profile_argo.py new file mode 100755 index 000000000..0e6b701ac --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/b2i/bufr2ioda_insitu_temp_profile_argo.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import sys +from b2iconverter.util import parse_arguments +from b2iconverter.bufr2ioda_config import Bufr2iodaConfig +from b2iconverter.bufr2ioda_converter import Bufr2ioda_Converter +from argo_ioda_variables import ArgoIODAVariables + + +platform_description = 'ARGO profiles from subpfl: temperature and salinity' + + +class ArgoConfig(Bufr2iodaConfig): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_temp_profile_argo.{self.cycle_datetime}.nc4" + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + + bufr2ioda_config = ArgoConfig( + script_name, + config_file, + platform_description) + + ioda_vars = ArgoIODAVariables() + ioda_vars.set_temperature_var_name("waterTemperature") + ioda_vars.set_temperature_error(0.02) + ioda_vars.set_salinity_var_name("salinity") + ioda_vars.set_salinity_error(0.01) + + argo = Bufr2ioda_Converter(bufr2ioda_config, ioda_vars, log_file) + + argo.run() + + if test_file: + result = argo.test(test_file) + sys.exit(result) diff --git a/ush/soca/prep_ocean_obs.py b/ush/soca/prep_ocean_obs.py index e751964ab..71f62416f 100644 --- a/ush/soca/prep_ocean_obs.py +++ b/ush/soca/prep_ocean_obs.py @@ -168,22 +168,37 @@ def initialize(self): obsprep_space['bufr2ioda converter'] = bufr2iodapy tmpl_filename = f"bufr2ioda_{obtype}.yaml" bufrconv_template = os.path.join(BUFR2IODA_TMPL_DIR, tmpl_filename) - output_files = [] # files to save to COM directory + input_files = [] # files to save to COM directory bufrconv_files = [] # files needed to populate the IODA converter config # for each cycle of the retrieved obs bufr files... for input_file, cycle in fetched_files: cycletime = cycle[8:10] ioda_filename = f"{RUN}.t{cycletime}z.{obs_space_name}.{cycle}.nc4" - output_files.append(ioda_filename) bufrconv_files.append((cycle, input_file, ioda_filename)) + input_files.append(ioda_filename) - obsprep_space['output file'] = output_files obsprep_space['bufrconv files'] = bufrconv_files + # set up config for concatenation + concat_config = { + 'provider': 'INSITUOBS', + 'window begin': obsprep_space['window begin'], + 'window end': obsprep_space['window end'], + 'variable': observer['obs space']['observed variables'][0], + 'error ratio': obsprep_space['error ratio'], + 'input files': input_files, + 'output file': f"{RUN}.t{cycletime}z.{obs_space_name}.{cdatestr}.nc4" + } + print('concat_config:', concat_config) + concat_config_file = obtype + '_concat.yaml' + + obsprep_space['output file'] = concat_config['output file'] + try: bufrconv = parse_j2yaml(bufrconv_template, bufrconv_config) bufrconv.update(obsprep_space) bufrconv.save(ioda_config_file) + save_as_yaml(concat_config, concat_config_file) except Exception as e: logger.warning(f"An exeception {e} occured while trying to create BUFR2IODA config") logger.warning(f"obtype {obtype} will be skipped") @@ -195,7 +210,7 @@ def initialize(self): obsprep_space['input files'] = [f[0] for f in fetched_files] ioda_filename = f"{RUN}.t{cyc:02d}z.{obs_space_name}.{cdatestr}.nc4" - obsprep_space['output file'] = [ioda_filename] + obsprep_space['output file'] = ioda_filename save_as_yaml(obsprep_space, ioda_config_file) obsspaces_to_convert.append({"obs space": obsprep_space}) @@ -238,7 +253,7 @@ def run(self): process = Process(target=prep_ocean_obs_utils.run_netcdf_to_ioda, args=(obs_space, self.task_config.OCNOBS2IODAEXEC)) elif obs_space["type"] == "bufr": - process = Process(target=prep_ocean_obs_utils.run_bufr_to_ioda, args=(obs_space,)) + process = Process(target=prep_ocean_obs_utils.run_bufr_to_ioda, args=(obs_space, self.task_config.OCNOBS2IODAEXEC)) else: logger.warning(f"Invalid observation format {obs_space['type']}, skipping obtype {obtype}") continue @@ -279,9 +294,8 @@ def finalize(self): conv_config_file_dest = os.path.join(COMOUT_OBS, conv_config_file) files_to_save.append([conv_config_file, conv_config_file_dest]) - for output_file in obs_space['output file']: - output_file_dest = os.path.join(COMOUT_OBS, output_file) - files_to_save.append([output_file, output_file_dest]) + output_file_dest = os.path.join(COMOUT_OBS, obs_space['output file']) + files_to_save.append([obs_space['output file'], output_file_dest]) try: FileHandler({'copy': files_to_save}).sync() diff --git a/ush/soca/prep_ocean_obs_utils.py b/ush/soca/prep_ocean_obs_utils.py index c8f42a375..7ac75b463 100755 --- a/ush/soca/prep_ocean_obs_utils.py +++ b/ush/soca/prep_ocean_obs_utils.py @@ -66,13 +66,16 @@ def run_netcdf_to_ioda(obsspace_to_convert, OCNOBS2IODAEXEC): return e.returncode -def run_bufr_to_ioda(obsspace_to_convert): - logger.info(f"running run_bufr_to_ioda on {obsspace_to_convert['name']}") +def run_bufr_to_ioda(obsspace_to_convert, OCNOBS2IODAEXEC): + obspace_name = obsspace_to_convert['name'] + logger.info(f"running run_bufr_to_ioda on {obspace_name}") bufrconv_yaml = obsspace_to_convert['conversion config file'] bufrconv_config = YAMLFile(bufrconv_yaml) + concat_config = YAMLFile(f'{obspace_name}_concat.yaml') bufr2iodapy = obsspace_to_convert['bufr2ioda converter'] obtype = obsspace_to_convert['name'] + # convert all the available bufr files to ioda for cycle, input_file, output_file in obsspace_to_convert['bufrconv files']: bufrconv_config['input_file'] = input_file bufrconv_config['output_file'] = output_file @@ -84,3 +87,14 @@ def run_bufr_to_ioda(obsspace_to_convert): except subprocess.CalledProcessError as e: logger.warning(f"bufr2ioda converter failed with error >{e}<, \ return code {e.returncode}") + return e.returncode + + # concatenate the ioda files to one ioda file with adjusted observation times + try: + subprocess.run([OCNOBS2IODAEXEC, f'{obspace_name}_concat.yaml'], check=True) + logger.info(f"ran ioda converter on obs space {obsspace_to_convert['name']} successfully") + return 0 + except subprocess.CalledProcessError as e: + logger.warning(f"ioda converter failed with error {e}, \ + return code {e.returncode}") + return e.returncode