From 9285f5f2a40529b549fffb696167150738c70b2c Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 9 Nov 2023 13:48:27 +0000 Subject: [PATCH 01/58] update prototype_3d observation list to process satwind_goes-16, update obsfile for satwnd.abi_goes-16 --- parm/atm/obs/config/satwind_goes-16.yaml | 2 +- parm/atm/obs/lists/gdas_prototype_3d.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/atm/obs/config/satwind_goes-16.yaml b/parm/atm/obs/config/satwind_goes-16.yaml index f5d7f3df6..33442b599 100644 --- a/parm/atm/obs/config/satwind_goes-16.yaml +++ b/parm/atm/obs/config/satwind_goes-16.yaml @@ -3,7 +3,7 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)satwind.abi_goes-16.{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)satwnd.abi_goes-16.tm00.nc obsdataout: engine: type: H5File diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml b/parm/atm/obs/lists/gdas_prototype_3d.yaml index 18d64e54e..ca478da42 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml @@ -1,9 +1,9 @@ observers: -- !INC ${OBS_YAML_DIR}/amsua_n19.yaml -- !INC ${OBS_YAML_DIR}/sondes.yaml +##- !INC ${OBS_YAML_DIR}/amsua_n19.yaml +##- !INC ${OBS_YAML_DIR}/sondes_prepbufr.yaml ##- !INC ${OBS_YAML_DIR}/atms_n20.yaml ##- !INC ${OBS_YAML_DIR}/aircraft.yaml -##- !INC ${OBS_YAML_DIR}/satwind.yaml +- !INC ${OBS_YAML_DIR}/satwind_goes-16.yaml ##- !INC ${OBS_YAML_DIR}/omi_aura.yaml ##- !INC ${OBS_YAML_DIR}/ompsnp_npp.yaml ##- !INC ${OBS_YAML_DIR}/ompstc8_npp.yaml From f5320c82b97d19cfe5ef546abdd8ccb144c00d53 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 9 Nov 2023 18:46:39 +0000 Subject: [PATCH 02/58] update GDAS/orion.lua to use spack-stack-1.5.1, bufr/12.0.1, and atlas/0.35.0 --- modulefiles/GDAS/orion.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modulefiles/GDAS/orion.lua b/modulefiles/GDAS/orion.lua index 8bff5b6c0..00ddecce0 100644 --- a/modulefiles/GDAS/orion.lua +++ b/modulefiles/GDAS/orion.lua @@ -6,7 +6,7 @@ local pkgName = myModuleName() local pkgVersion = myModuleVersion() local pkgNameVer = myModuleFullName() -prepend_path("MODULEPATH", '/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core') +prepend_path("MODULEPATH", '/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core') prepend_path("MODULEPATH", '/work2/noaa/da/python/opt/modulefiles/stack') -- below two lines get us access to the spack-stack modules @@ -27,7 +27,7 @@ load("nco/5.0.6") load("parallelio/2.5.10") load("wget/1.14") load("boost/1.78") -load("bufr/12.0.0") +load("bufr/12.0.1") load("git-lfs/2.12.0") load("ecbuild/3.7.2") load("openjpeg/2.3.1") @@ -39,7 +39,7 @@ load("fftw/3.3.10") load("fckit/0.11.0") load("fiat/1.2.0") load("ectrans/1.2.0") -load("atlas/0.34.0") +load("atlas/0.35.0") load("sp/2.3.3") load("gsl-lite/0.37.0") load("libjpeg/2.1.0") From 8c3496614f65311f1bedbd41085a4a013eab79f3 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Sat, 11 Nov 2023 16:20:17 -0600 Subject: [PATCH 03/58] Remove a double-declaired "obs post filters" in satwnd GOES YAMLs. --- parm/atm/obs/config/satwind_goes-16.yaml | 1 - parm/atm/obs/config/satwind_goes-17.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/parm/atm/obs/config/satwind_goes-16.yaml b/parm/atm/obs/config/satwind_goes-16.yaml index 33442b599..e1dbd2ff7 100644 --- a/parm/atm/obs/config/satwind_goes-16.yaml +++ b/parm/atm/obs/config/satwind_goes-16.yaml @@ -427,7 +427,6 @@ obs post filters: action: name: reject -obs post filters: # Reject GOES (247) when difference of wind direction is more than 50 degrees. # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - filter: Bounds Check diff --git a/parm/atm/obs/config/satwind_goes-17.yaml b/parm/atm/obs/config/satwind_goes-17.yaml index b413b969b..c363acf5d 100644 --- a/parm/atm/obs/config/satwind_goes-17.yaml +++ b/parm/atm/obs/config/satwind_goes-17.yaml @@ -427,7 +427,6 @@ obs post filters: action: name: reject -obs post filters: # Reject GOES (247) when difference of wind direction is more than 50 degrees. # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - filter: Bounds Check From e1a537529bf7adabd072d6d27437ba0e92513402 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:28:47 -0500 Subject: [PATCH 04/58] Modify BUFR converter and filter yamls for GOES AMV (#724) Two updates for GOES AMV 1. BUFR Converter: - remove wind direction and speed - change sensorZenithAngle to satelliteZenithAngle 2. UFO Filters: - change `sensorZenithAngle` to `satelliteZenithAngle` - add `linear obs operator` section The declaration of `linear obs operator" is necessary for GOES AMV since there is no linearized component for applying the 10-meter factor. The increments look reasonable: ``` 0: ---------------------------------------------------------------------------------------------------- 0: Increment print | number of fields = 8 | cube sphere face size: C768 0: eastward_wind | Min:-1.642335e-01 Max:+1.660654e-01 RMS:+2.746685e-04 0: northward_wind | Min:-2.120068e-01 Max:+1.663539e-01 RMS:+2.631295e-04 0: air_temperature | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: surface_pressure | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: specific_humidity | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: cloud_liquid_ice | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: cloud_liquid_water | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: ozone_mass_mixing_ratio | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: ---------------------------------------------------------------------------------------------------- ``` The UFO filter results vs. GSI -- looks good ![ufo_abi_goes-16_omf_windEastward_qc_time1](https://github.com/NOAA-EMC/GDASApp/assets/36091766/017aeabc-d6ca-4643-b037-16f46dde76a0) ![gsi_abi_goes-16_omf_windEastward_qc_time1](https://github.com/NOAA-EMC/GDASApp/assets/36091766/d99599a2-3f36-4ef6-aa2e-4611500b8c43) --- parm/atm/obs/config/satwind_goes-16.yaml | 5 ++++- parm/atm/obs/config/satwind_goes-17.yaml | 5 ++++- .../bufr2ioda/bufr2ioda_satwind_amv_goes.py | 20 ++++--------------- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/parm/atm/obs/config/satwind_goes-16.yaml b/parm/atm/obs/config/satwind_goes-16.yaml index e1dbd2ff7..ebdc35b3a 100644 --- a/parm/atm/obs/config/satwind_goes-16.yaml +++ b/parm/atm/obs/config/satwind_goes-16.yaml @@ -17,6 +17,9 @@ obs operator: hofx scaling field: SurfaceWindScalingPressure hofx scaling field group: DerivedVariables +linear obs operator: + name: VertInterp + obs prior filters: # Apply variable changes needed for wind scaling # For wind observations with pressure provided @@ -181,7 +184,7 @@ obs post filters: - name: windEastward - name: windNorthward test variables: - - name: MetaData/sensorZenithAngle + - name: MetaData/satelliteZenithAngle maxvalue: 68. action: name: reject diff --git a/parm/atm/obs/config/satwind_goes-17.yaml b/parm/atm/obs/config/satwind_goes-17.yaml index c363acf5d..c234780f0 100644 --- a/parm/atm/obs/config/satwind_goes-17.yaml +++ b/parm/atm/obs/config/satwind_goes-17.yaml @@ -17,6 +17,9 @@ obs operator: hofx scaling field: SurfaceWindScalingPressure hofx scaling field group: DerivedVariables +obs linear operator: + name: VertInterp + obs prior filters: # Apply variable changes needed for wind scaling # For wind observations with pressure provided @@ -181,7 +184,7 @@ obs post filters: - name: windEastward - name: windNorthward test variables: - - name: MetaData/sensorZenithAngle + - name: MetaData/satelliteZenithAngle maxvalue: 68. action: name: reject diff --git a/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.py b/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.py index 88f0d7a5b..7e1207f54 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.py @@ -124,7 +124,7 @@ def bufr_to_ioda(config, logger): q.add('hour', '*/HOUR') q.add('minute', '*/MINU') q.add('second', '*/SECO') - q.add('sensorZenithAngle', '*/SAZA') + q.add('satelliteZenithAngle', '*/SAZA') q.add('sensorCentralFrequency', '*/SCCF') q.add('pressure', '*/PRLC[1]') @@ -173,7 +173,7 @@ def bufr_to_ioda(config, logger): second = r.get('second') lat = r.get('latitude') lon = r.get('longitude') - satzenang = r.get('sensorZenithAngle') + satzenang = r.get('satelliteZenithAngle') pressure = r.get('pressure', type='float') chanfreq = r.get('sensorCentralFrequency', type='float') @@ -357,10 +357,10 @@ def bufr_to_ioda(config, logger): .write_data(satid2) # Sensor Zenith Angle - obsspace.create_var('MetaData/sensorZenithAngle', dtype=satzenang2.dtype, fillval=satzenang2.fill_value) \ + obsspace.create_var('MetaData/satelliteZenithAngle', dtype=satzenang2.dtype, fillval=satzenang2.fill_value) \ .write_attr('units', 'degree') \ .write_attr('valid_range', np.array([0, 90], dtype=np.float32)) \ - .write_attr('long_name', 'Sensor Zenith Angle') \ + .write_attr('long_name', 'Satellite Zenith Angle') \ .write_data(satzenang2) # Sensor Centrall Frequency @@ -428,18 +428,6 @@ def bufr_to_ioda(config, logger): .write_attr('long_name', 'Station Elevation') \ .write_data(stnelev2) - # Wind Speed - obsspace.create_var('ObsValue/windSpeed', dtype=wspd2.dtype, fillval=wspd2.fill_value) \ - .write_attr('units', 'm s-1') \ - .write_attr('long_name', 'Wind Speed') \ - .write_data(wspd2) - - # Wind Direction - obsspace.create_var('ObsValue/windDirection', dtype=wdir2.dtype, fillval=wdir2.fill_value) \ - .write_attr('units', 'degrees') \ - .write_attr('long_name', 'Wind Direction') \ - .write_data(wdir2) - # U-Wind Component obsspace.create_var('ObsValue/windEastward', dtype=uob2.dtype, fillval=wspd2.fill_value) \ .write_attr('units', 'm s-1') \ From dda513fa46bafbfcd63af8a3e532aa387cef73bf Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 14 Nov 2023 13:35:54 -0500 Subject: [PATCH 05/58] Make the python bufr2ioda driver more generic (#725) Rather than each new ob type being manually added to the list, we can use `glob.glob` to get the list of available observation types. --- ush/ioda/bufr2ioda/run_bufr2ioda.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 34df54b5f..c567f7ae9 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import argparse +import glob import os from pathlib import Path from gen_bufr2ioda_json import gen_bufr_json @@ -30,8 +31,9 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): } # Specify observation types to be processed by a script - BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr", "adpsfc_prepbufr", "sfcshp_prepbufr", "acft_profiles_prepbufr", - "gpsro_bufr", "conventional_prepbufr_ps"] + BUFR_py_files = glob.glob(os.path.join(USH_IODA, 'bufr2ioda_*.py')) + BUFR_py_files = [os.path.basename(f) for f in BUFR_py_files] + BUFR_py = [f.replace('bufr2ioda_', '').replace('.py', '') for f in BUFR_py_files] for obtype in BUFR_py: logger.info(f"Convert {obtype}...") From 6e6a48049704e7f826e52a7872e6fe384401d400 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Wed, 15 Nov 2023 14:55:00 -0500 Subject: [PATCH 06/58] Modify BUFR converter and filter yamls for MetoOp Scatwind (#731) Two updates for MetOp SCATWIND; one minor updates for GOES AMV 1. BUFR Converter: - remove wind direction and speed - add `stationElevation` as zero 2. UFO Filters: - add linear obs operator section The declaration of `linear obs operator" is necessary for GOES AMV since there is no linearized component for applying the 10-meter factor. - add `Gaussian Thinning` filter to thin data with 75 km box (equal box size, prefer data close to central of the box) 3. For GOES AMV YAML files, change output data extension from `nc4` to `nc` UFO vs. GSI O-F and QC comparison: O-F and data count passed look close ![ufo_ascat_metop-b_omf_windEastward_qc (1)](https://github.com/NOAA-EMC/GDASApp/assets/36091766/f234543e-e0b5-4c6e-add9-3c7365c0c469) ![gsi_ascat_metop-b_omf_windEastward_qc](https://github.com/NOAA-EMC/GDASApp/assets/36091766/37f4fba8-bd2a-4a54-b45e-4055e3ef3142) 3Dvar increments are reasonable with MetOp-B SCATWIND alone. ``` 0: ---------------------------------------------------------------------------------------------------- 0: Increment print | number of fields = 8 | cube sphere face size: C768 0: eastward_wind | Min:-1.741092e+00 Max:+1.712287e+00 RMS:+2.282584e-03 0: northward_wind | Min:-1.825957e+00 Max:+1.930385e+00 RMS:+2.496075e-03 0: air_temperature | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: surface_pressure | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: specific_humidity | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: cloud_liquid_ice | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: cloud_liquid_water | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: ozone_mass_mixing_ratio | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: ---------------------------------------------------------------------------------------------------- ``` 3DVar increments are also reasonable with MetOp-B + MetOp-A assimilated --- parm/atm/obs/config/satwind_goes-16.yaml | 2 +- parm/atm/obs/config/satwind_goes-17.yaml | 2 +- parm/atm/obs/config/scatwind_metop-a.yaml | 18 ++++++++++++++-- parm/atm/obs/config/scatwind_metop-b.yaml | 18 ++++++++++++++-- ush/ioda/bufr2ioda/bufr2ioda_satwind_scat.py | 22 +++++++++----------- 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/parm/atm/obs/config/satwind_goes-16.yaml b/parm/atm/obs/config/satwind_goes-16.yaml index ebdc35b3a..c35f9626b 100644 --- a/parm/atm/obs/config/satwind_goes-16.yaml +++ b/parm/atm/obs/config/satwind_goes-16.yaml @@ -7,7 +7,7 @@ obs space: obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_satwind_abi_goes-16_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_satwind_abi_goes-16_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] diff --git a/parm/atm/obs/config/satwind_goes-17.yaml b/parm/atm/obs/config/satwind_goes-17.yaml index c234780f0..65bf9194e 100644 --- a/parm/atm/obs/config/satwind_goes-17.yaml +++ b/parm/atm/obs/config/satwind_goes-17.yaml @@ -7,7 +7,7 @@ obs space: obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_satwind_abi_goes-17_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_satwind_abi_goes-17_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] diff --git a/parm/atm/obs/config/scatwind_metop-a.yaml b/parm/atm/obs/config/scatwind_metop-a.yaml index 4735c31fe..9dcbe9eab 100644 --- a/parm/atm/obs/config/scatwind_metop-a.yaml +++ b/parm/atm/obs/config/scatwind_metop-a.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)ascatw.ascat_metop-a.{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ascatw.ascat_metop-a.tm00.nc obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_ascatw_ascat_metop-a_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_ascatw_ascat_metop-a_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] @@ -23,6 +23,20 @@ obs operator: hofx scaling field: SurfaceWindScalingHeight hofx scaling field group: DerivedVariables +linear obs operator: + name: VertInterp + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + +obs pre filters: +- filter: Gaussian Thinning + horizontal_mesh: 75 + use_reduced_horizontal_grid: true + round_horizontal_bin_count_to_nearest: true + partition_longitude_bins_using_mesh: true + obs prior filters: # Apply variable changes needed for rescaled height coordinate - filter: Variable Transforms diff --git a/parm/atm/obs/config/scatwind_metop-b.yaml b/parm/atm/obs/config/scatwind_metop-b.yaml index 5ed7c38bc..16f10351b 100644 --- a/parm/atm/obs/config/scatwind_metop-b.yaml +++ b/parm/atm/obs/config/scatwind_metop-b.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)ascatw.ascat_metop-b.{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ascatw.ascat_metop-b.tm00.nc obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_ascatw_ascat_metop-b_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_ascatw_ascat_metop-b_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] @@ -23,6 +23,20 @@ obs operator: hofx scaling field: SurfaceWindScalingHeight hofx scaling field group: DerivedVariables +linear obs operator: + name: VertInterp + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + +obs pre filters: +- filter: Gaussian Thinning + horizontal_mesh: 75 + use_reduced_horizontal_grid: true + round_horizontal_bin_count_to_nearest: true + partition_longitude_bins_using_mesh: true + obs prior filters: # Apply variable changes needed for rescaled height coordinate - filter: Variable Transforms diff --git a/ush/ioda/bufr2ioda/bufr2ioda_satwind_scat.py b/ush/ioda/bufr2ioda/bufr2ioda_satwind_scat.py index 352dac6a8..abdeb6358 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_satwind_scat.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_satwind_scat.py @@ -169,6 +169,9 @@ def bufr_to_ioda(config, logger): logger.debug('Creating derived variables - observation height') height = np.full_like(lat, 10.) + logger.debug('Creating derived variables - station elevation') + stnelv = np.full_like(lat, 0.) + end_time = time.time() running_time = end_time - start_time logger.debug(f'Processing time for creating derived variables : {running_time} seconds') @@ -209,6 +212,7 @@ def bufr_to_ioda(config, logger): timestamp2 = timestamp[mask] pressure2 = pressure[mask] height2 = height[mask] + stnelv2 = stnelv[mask] obstype2 = obstype[mask] # QC Info @@ -299,6 +303,12 @@ def bufr_to_ioda(config, logger): .write_attr('long_name', 'Height of Observation') \ .write_data(height2) + # Station Elevation + obsspace.create_var('MetaData/stationElevation', dtype=stnelv2.dtype, fillval=stnelv2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Station Elevation') \ + .write_data(stnelv2) + # ObsType based on sensor type obsspace.create_var('ObsType/windEastward', dtype=obstype2.dtype, fillval=obstype2.fill_value) \ .write_attr('long_name', 'PrepBUFR Report Type') \ @@ -309,18 +319,6 @@ def bufr_to_ioda(config, logger): .write_attr('long_name', 'PrepBUFR Report Type') \ .write_data(obstype2) - # Wind Speed - obsspace.create_var('ObsValue/windSpeed', dtype=wspd2.dtype, fillval=wspd2.fill_value) \ - .write_attr('units', 'm s-1') \ - .write_attr('long_name', 'Wind Speed at 10 Meters') \ - .write_data(wspd2) - - # Wind Direction - obsspace.create_var('ObsValue/windDirection', dtype=wdir2.dtype, fillval=wdir2.fill_value) \ - .write_attr('units', 'degrees') \ - .write_attr('long_name', 'Wind Direction at 10 Meters') \ - .write_data(wdir2) - # U-Wind Component obsspace.create_var('ObsValue/windEastward', dtype=uob2.dtype, fillval=uob2.fill_value) \ .write_attr('units', 'm s-1') \ From 5c305aabe817d52f40ecdaf034bb0de1841e0815 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 15 Nov 2023 15:08:01 -0500 Subject: [PATCH 07/58] Have the driver script process both python scripts and YAMLs + bufr2ioda.x (#730) This PR allows for all obtypes to be processed in two loops. We still need to figure out a way to process these in parallel (somewhat). --- parm/ioda/bufr2ioda/atms_beamwidth.txt | 36 +++++++++++ .../bufr2ioda_atms.yaml} | 7 ++- ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py | 60 ++++++------------- ush/ioda/bufr2ioda/run_bufr2ioda.py | 39 +++++++++++- 4 files changed, 95 insertions(+), 47 deletions(-) create mode 100644 parm/ioda/bufr2ioda/atms_beamwidth.txt rename parm/ioda/{bufr_atms.yaml => bufr2ioda/bufr2ioda_atms.yaml} (95%) diff --git a/parm/ioda/bufr2ioda/atms_beamwidth.txt b/parm/ioda/bufr2ioda/atms_beamwidth.txt new file mode 100644 index 000000000..21a002d0c --- /dev/null +++ b/parm/ioda/bufr2ioda/atms_beamwidth.txt @@ -0,0 +1,36 @@ +# LOC = data/preproc/atms_beamwidth.dat +# WMO Satellite ID +224 +# Version identifier +1 +# Sampling distance (deg) +1.11 +# Number of channels to modify +22 +# Channel, beam width, new width, cutoff, nxaverage, nyaverage, QC dist +# Note: to use FFT technique, set new width and cutoff as required and set +# nxaverage and nyaverage to 0. To use simple averaging set new width to 0.0 +# and use nxaverage and nyaverage (e.g. 3 3 for 3x3 averaging). +# QC Dist gives number of points around missing value to flag as missing. + 1 5.2 3.33 0.4 0 0 11 + 2 5.2 3.33 0.4 0 0 11 + 3 2.2 3.33 0.0 0 0 5 + 4 2.2 3.33 0.0 0 0 5 + 5 2.2 3.33 0.0 0 0 5 + 6 2.2 3.33 0.0 0 0 5 + 7 2.2 3.33 0.0 0 0 5 + 8 2.2 3.33 0.0 0 0 5 + 9 2.2 3.33 0.0 0 0 5 +10 2.2 3.33 0.0 0 0 5 +11 2.2 3.33 0.0 0 0 5 +12 2.2 3.33 0.0 0 0 5 +13 2.2 3.33 0.0 0 0 5 +14 2.2 3.33 0.0 0 0 5 +15 2.2 3.33 0.0 0 0 5 +16 2.2 3.33 0.0 0 0 5 +17 1.1 3.33 0.0 0 0 5 +18 1.1 3.33 0.0 0 0 5 +19 1.1 3.33 0.0 0 0 5 +20 1.1 3.33 0.0 0 0 5 +21 1.1 3.33 0.0 0 0 5 +22 1.1 3.33 0.0 0 0 5 diff --git a/parm/ioda/bufr_atms.yaml b/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml similarity index 95% rename from parm/ioda/bufr_atms.yaml rename to parm/ioda/bufr2ioda/bufr2ioda_atms.yaml index 7e8738e2f..39fa399ab 100644 --- a/parm/ioda/bufr_atms.yaml +++ b/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml @@ -1,7 +1,7 @@ observations: - obs space: name: bufr - obsdatain: "$(BUFR_in)" + obsdatain: "{{ DMPDIR }}/{{ RUN }}.{{ PDY }}/{{ cyc }}/atmos/{{ RUN }}.t{{ cyc }}z.atms.tm00.bufr_d" exports: variables: @@ -51,9 +51,9 @@ observations: sensorScanAngle: sensorScanAngle: fieldOfViewNumber: "*/FOVN" - heightOfStation: "*/HMSL" scanStart: -52.725 scanStep: 1.110 + sensor: atms sensorChannelNumber: query: "*/ATMSCH/CHNM" @@ -80,10 +80,11 @@ observations: map: _224: npp _225: n20 +# _226: n21 ioda: backend: netcdf - obsdataout: "$(IODA_out)" + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.atms.$(splitvar).tm00.nc" dimensions: - name: Channel diff --git a/ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py b/ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py index cd0dd1ce6..ce7b115ce 100755 --- a/ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py +++ b/ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py @@ -5,52 +5,30 @@ # and certain configuration parameters import argparse import os -from wxflow import Template, TemplateConstants, YAMLFile +from wxflow import Logger, parse_j2yaml, cast_strdict_as_dtypedict, save_as_yaml +from wxflow import Template, TemplateConstants +# initialize root logger +logger = Logger('gen_bufr2ioda_yaml.py', level='INFO', colored_log=True) -# list of satellite radiance BUFR files that need split by SatId -sat_list = [ - 'atms', - '1bamua', - '1bmhs', - 'crisf4', - 'iasidb', -] - -def gen_bufr_yaml(config): - # open the template input file - bufr_yaml = YAMLFile(path=config['template yaml']) - # determine if splits need in the output file path - obtype = config['obtype'] - if obtype in sat_list: - # split by satellite platform - obtype_out = f"{obtype}_{{splits/satId}}" - else: - obtype_out = obtype - # construct the output IODA file path - output_ioda = [ - config['run'], - f"t{config['cyc']:02}z", - obtype_out, - 'nc', - ] - output_ioda_str = '.'.join(output_ioda) - output_ioda_file = os.path.join(config['output dir'], output_ioda_str) - # construct the template substitution dict - substitutions = { - 'BUFR_in': config['input file'], - 'IODA_out': output_ioda_file, - } - # substitue templates - bufr_yaml = Template.substitute_structure(bufr_yaml, TemplateConstants.DOLLAR_PARENTHESES, substitutions.get) - # write out BUFR converter YAML file - bufr_yaml.save(config['output yaml file']) +def gen_bufr_yaml(config, template, output): + # read in templated YAML and do substitution + logger.info(f"Using {template} as input") + bufr_config = parse_j2yaml(template, config) + # need to do some special manipulation for the splits + substitutions = {'splitvar': '{splits/satId}'} + bufr_config = Template.substitute_structure(bufr_config, TemplateConstants.DOLLAR_PARENTHESES, substitutions.get) + save_as_yaml(bufr_config, output) + logger.info(f"Wrote to {output}") if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-c', '--config', type=str, help='Input YAML Configuration', required=True) + parser.add_argument('-t', '--template', type=str, help='Input YAML template', required=True) + parser.add_argument('-o', '--output', type=str, help='Output YAML file', required=True) args = parser.parse_args() - config = YAMLFile(path=args.config) - gen_bufr_yaml(config) + # get the config from your environment + config = cast_strdict_as_dtypedict(os.environ) + # call the parsing function + gen_bufr_yaml(config, args.template, args.output) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index c567f7ae9..c5ab09517 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -2,8 +2,10 @@ import argparse import glob import os +import shutil from pathlib import Path from gen_bufr2ioda_json import gen_bufr_json +from gen_bufr2ioda_yaml import gen_bufr_yaml from wxflow import (Logger, Executable, cast_as_dtype, logit, to_datetime, datetime_to_YMDH, Task, rm_p) @@ -18,6 +20,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # Get gdasapp root directory DIR_ROOT = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../..")) USH_IODA = os.path.join(DIR_ROOT, "ush", "ioda", "bufr2ioda") + BIN_GDAS = os.path.join(DIR_ROOT, "build", "bin") # Create output directory if it doesn't exist os.makedirs(COM_OBS, exist_ok=True) @@ -27,14 +30,21 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): 'RUN': RUN, 'current_cycle': current_cycle, 'DMPDIR': DMPDIR, - 'COM_OBS': COM_OBS + 'COM_OBS': COM_OBS, + 'PDY': current_cycle.strftime('%Y%m%d'), + 'cyc': current_cycle.strftime('%H'), } + # copy necessary fix files to runtime directory + shutil.copy(os.path.join(config_template_dir, "atms_beamwidth.txt"), + os.path.join(os.getcwd(), "atms_beamwidth.txt")) + # Specify observation types to be processed by a script BUFR_py_files = glob.glob(os.path.join(USH_IODA, 'bufr2ioda_*.py')) BUFR_py_files = [os.path.basename(f) for f in BUFR_py_files] BUFR_py = [f.replace('bufr2ioda_', '').replace('.py', '') for f in BUFR_py_files] + # NOTE or TODO - how to parallelize these loops???? for obtype in BUFR_py: logger.info(f"Convert {obtype}...") json_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") @@ -51,8 +61,31 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): cmd() # Check if the converter was successful - if os.path.exists(json_output_file): - rm_p(json_output_file) + # if os.path.exists(json_output_file): + # rm_p(json_output_file) + + # Specify observation types to be processed by the bufr2ioda executable + BUFR_yaml_files = glob.glob(os.path.join(config_template_dir, '*.yaml')) + BUFR_yaml_files = [os.path.basename(f) for f in BUFR_yaml_files] + BUFR_yaml = [f.replace('bufr2ioda_', '').replace('.yaml', '') for f in BUFR_yaml_files] + + for obtype in BUFR_yaml: + logger.info(f"Convert {obtype}...") + yaml_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.yaml") + filename = 'bufr2ioda_' + obtype + '.yaml' + template = os.path.join(config_template_dir, filename) + gen_bufr_yaml(config, template, yaml_output_file) + + # use the bufr2ioda executable for the ob type + bufr2iodaexe = BIN_GDAS + '/bufr2ioda.x' + cmd = Executable(bufr2iodaexe) + cmd.add_default_arg(yaml_output_file) + logger.info(f"Executing {cmd}") + cmd() + + # Check if the converter was successful + # if os.path.exists(yaml_output_file): + # rm_p(yaml_output_file) if __name__ == "__main__": From 9fc4d739a1857a341feeb8d866733828540983cd Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 16 Nov 2023 08:34:25 -0500 Subject: [PATCH 08/58] Use parallel processing to speed up obs processing (#733) Using python multiprocessing to generate obs in parallel. The current list of obs goes from 10+ minutes to completing in ~5.5 minutes. --- ush/ioda/bufr2ioda/run_bufr2ioda.py | 48 +++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index c5ab09517..2976cc241 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 import argparse import glob +import multiprocessing as mp import os import shutil +from itertools import repeat from pathlib import Path from gen_bufr2ioda_json import gen_bufr_json from gen_bufr2ioda_yaml import gen_bufr_yaml @@ -12,6 +14,24 @@ # Initialize root logger logger = Logger('run_bufr2ioda.py', level='INFO', colored_log=True) +# get parallel processing info +num_cores = mp.cpu_count() + + +def mp_bufr_py(script, infile): + cmd = Executable(script) + cmd.add_default_arg('-c') + cmd.add_default_arg(infile) + logger.info(f"Executing {cmd}") + cmd() + + +def mp_bufr_yaml(bufr2iodaexe, yamlfile): + cmd = Executable(bufr2iodaexe) + cmd.add_default_arg(yamlfile) + logger.info(f"Executing {cmd}") + cmd() + @logit(logger) def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): @@ -44,7 +64,8 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): BUFR_py_files = [os.path.basename(f) for f in BUFR_py_files] BUFR_py = [f.replace('bufr2ioda_', '').replace('.py', '') for f in BUFR_py_files] - # NOTE or TODO - how to parallelize these loops???? + json_files = [] + scripts = [] for obtype in BUFR_py: logger.info(f"Convert {obtype}...") json_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") @@ -54,21 +75,25 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # Use the converter script for the ob type bufr2iodapy = USH_IODA + '/bufr2ioda_' + obtype + ".py" - cmd = Executable(bufr2iodapy) - cmd.add_default_arg('-c') - cmd.add_default_arg(json_output_file) - logger.info(f"Executing {cmd}") - cmd() + + # append the values to the lists + json_files.append(json_output_file) + scripts.append(bufr2iodapy) # Check if the converter was successful # if os.path.exists(json_output_file): # rm_p(json_output_file) + # run all python scripts in parallel + with mp.Pool(num_cores) as pool: + pool.starmap(mp_bufr_py, zip(scripts, json_files)) + # Specify observation types to be processed by the bufr2ioda executable BUFR_yaml_files = glob.glob(os.path.join(config_template_dir, '*.yaml')) BUFR_yaml_files = [os.path.basename(f) for f in BUFR_yaml_files] BUFR_yaml = [f.replace('bufr2ioda_', '').replace('.yaml', '') for f in BUFR_yaml_files] + yaml_files = [] for obtype in BUFR_yaml: logger.info(f"Convert {obtype}...") yaml_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.yaml") @@ -78,15 +103,18 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # use the bufr2ioda executable for the ob type bufr2iodaexe = BIN_GDAS + '/bufr2ioda.x' - cmd = Executable(bufr2iodaexe) - cmd.add_default_arg(yaml_output_file) - logger.info(f"Executing {cmd}") - cmd() + + # append the values to the lists + yaml_files.append(yaml_output_file) # Check if the converter was successful # if os.path.exists(yaml_output_file): # rm_p(yaml_output_file) + # run all bufr2ioda yamls in parallel + with mp.Pool(num_cores) as pool: + pool.starmap(mp_bufr_yaml, zip(repeat(bufr2iodaexe), yaml_files)) + if __name__ == "__main__": parser = argparse.ArgumentParser(description='Convert bufr dump file to ioda format') From 8f6b9c15570faf17d4a05f1094355edd918fc304 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Thu, 16 Nov 2023 11:40:13 -0600 Subject: [PATCH 09/58] temp --- CMakeLists.txt | 2 +- parm/atm/variational/3dvar_dripcg.yaml | 36 +------------------------- ush/ioda/bufr2ioda/run_bufr2ioda.py | 2 +- 3 files changed, 3 insertions(+), 37 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b26b64b5b..dfee7c7ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,7 +107,7 @@ if(BUILD_GDASBUNDLE) # Build IODA converters option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON) if(BUILD_IODA_CONVERTERS) - ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" BRANCH develop ) + ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" ) endif() # Land associated repositories diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml index abd945842..2f9ccb76f 100644 --- a/parm/atm/variational/3dvar_dripcg.yaml +++ b/parm/atm/variational/3dvar_dripcg.yaml @@ -33,7 +33,7 @@ variational: minimizer: algorithm: DRIPCG iterations: - - ninner: 2 + - ninner: 1 gradient norm reduction: 1e-10 test: on geometry: @@ -50,40 +50,6 @@ variational: field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml diagnostics: departures: bkgmob - - ninner: 4 - gradient norm reduction: 1e-10 - test: on - geometry: - fms initialization: - namelist filename: ./fv3jedi/fmsmpp.nml - field table filename: ./fv3jedi/field_table - akbk: ./fv3jedi/akbk.nc4 - layout: - - $(layout_x) - - $(layout_y) - npx: $(npx_anl) - npy: $(npy_anl) - npz: $(npz_anl) - field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml - diagnostics: - departures: bkgmob1 final: diagnostics: departures: anlmob - increment: - output: - filetype: auxgrid - gridtype: gaussian - filename: ./anl/atminc. - geometry: - fms initialization: - namelist filename: ./fv3jedi/fmsmpp.nml - field table filename: ./fv3jedi/field_table - akbk: ./fv3jedi/akbk.nc4 - layout: - - $(layout_x) - - $(layout_y) - npx: $(npx_anl) - npy: $(npy_anl) - npz: $(npz_anl) - field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index da01dd131..1130658b1 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -32,7 +32,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # Specify observation types to be processed by a script BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr", "adpsfc_prepbufr", "sfcshp_prepbufr"] - + BUFR_py = ["ncep_amsua",] for obtype in BUFR_py: logger.info(f"Convert {obtype}...") json_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") From 0970ec87c22778f5df1f325fec9d230703eebdf2 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 17 Nov 2023 09:29:20 -0500 Subject: [PATCH 10/58] Fix YAML so minimization works for satwind G17 (#743) @RussTreadon-NOAA noted that `obs linear operator` should be `linear obs operator` in the GOES-17 satwind YAML. This fixes that. --- parm/atm/obs/config/satwind_goes-17.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/atm/obs/config/satwind_goes-17.yaml b/parm/atm/obs/config/satwind_goes-17.yaml index 65bf9194e..19eee100a 100644 --- a/parm/atm/obs/config/satwind_goes-17.yaml +++ b/parm/atm/obs/config/satwind_goes-17.yaml @@ -17,7 +17,7 @@ obs operator: hofx scaling field: SurfaceWindScalingPressure hofx scaling field group: DerivedVariables -obs linear operator: +linear obs operator: name: VertInterp obs prior filters: From ae224668140e75a9fb03c2699f6778b4407edebd Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 17 Nov 2023 13:19:58 -0500 Subject: [PATCH 11/58] Consolidate parallel processing (#745) This reduces runtime further to run everything in one parallel pool. Now, things complete in under 4 minutes (as of now). --- ush/ioda/bufr2ioda/run_bufr2ioda.py | 37 +++++++++++------------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 2976cc241..45dad6539 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -18,17 +18,12 @@ num_cores = mp.cpu_count() -def mp_bufr_py(script, infile): - cmd = Executable(script) - cmd.add_default_arg('-c') - cmd.add_default_arg(infile) - logger.info(f"Executing {cmd}") - cmd() - - -def mp_bufr_yaml(bufr2iodaexe, yamlfile): - cmd = Executable(bufr2iodaexe) - cmd.add_default_arg(yamlfile) +def mp_bufr_converter(exename, configfile): + cmd = Executable(exename) + filetype = Path(configfile).suffix + if filetype == '.json': + cmd.add_default_arg('-c') + cmd.add_default_arg(configfile) logger.info(f"Executing {cmd}") cmd() @@ -64,8 +59,8 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): BUFR_py_files = [os.path.basename(f) for f in BUFR_py_files] BUFR_py = [f.replace('bufr2ioda_', '').replace('.py', '') for f in BUFR_py_files] - json_files = [] - scripts = [] + config_files = [] + exename = [] for obtype in BUFR_py: logger.info(f"Convert {obtype}...") json_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") @@ -77,23 +72,18 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): bufr2iodapy = USH_IODA + '/bufr2ioda_' + obtype + ".py" # append the values to the lists - json_files.append(json_output_file) - scripts.append(bufr2iodapy) + config_files.append(json_output_file) + exename.append(bufr2iodapy) # Check if the converter was successful # if os.path.exists(json_output_file): # rm_p(json_output_file) - # run all python scripts in parallel - with mp.Pool(num_cores) as pool: - pool.starmap(mp_bufr_py, zip(scripts, json_files)) - # Specify observation types to be processed by the bufr2ioda executable BUFR_yaml_files = glob.glob(os.path.join(config_template_dir, '*.yaml')) BUFR_yaml_files = [os.path.basename(f) for f in BUFR_yaml_files] BUFR_yaml = [f.replace('bufr2ioda_', '').replace('.yaml', '') for f in BUFR_yaml_files] - yaml_files = [] for obtype in BUFR_yaml: logger.info(f"Convert {obtype}...") yaml_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.yaml") @@ -105,15 +95,16 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): bufr2iodaexe = BIN_GDAS + '/bufr2ioda.x' # append the values to the lists - yaml_files.append(yaml_output_file) + config_files.append(yaml_output_file) + exename.append(bufr2iodaexe) # Check if the converter was successful # if os.path.exists(yaml_output_file): # rm_p(yaml_output_file) - # run all bufr2ioda yamls in parallel + # run everything in parallel with mp.Pool(num_cores) as pool: - pool.starmap(mp_bufr_yaml, zip(repeat(bufr2iodaexe), yaml_files)) + pool.starmap(mp_bufr_converter, zip(exename, config_files)) if __name__ == "__main__": From d331a25d4a11017691cac769dd6362fb8c4e121a Mon Sep 17 00:00:00 2001 From: BrettHoover-NOAA <98188219+BrettHoover-NOAA@users.noreply.github.com> Date: Fri, 17 Nov 2023 12:53:12 -0600 Subject: [PATCH 12/58] Add AHI/Himawari-8 assimilation to end-to-end GDASApp validation (#746) Adding satwinds from the Advanced Himawari Imager (AHI) from Himawari-8 to GDASApp end-to-end testing new files include: parm/atm/obs/config/satwind_ahi_h8.yaml: QC filter YAML for AHI Himawari-8 satwinds parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.json: JSON containing data format, sensor, and satellite information for AHI Himawari-8 satwinds ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.py: bufr2ioda code for extracting AHI Himawari-8 satwinds from BUFR modified files include: ush/ioda/bufr2ioda/run_bufr2ioda.py: added `"satwind_amv_ahi"` to list `BUFR_py` See #741 for details on testing. JEDI/GSI comparisons look good with GSI thinning turned off, but there are large ob-count disparities both before and after QC when comparing GSI+thinning to JEDI+thinning. The tested `Gaussian Thinning` filter is included in the YAML but commented out with a note. --------- Co-authored-by: Brett Hoover Co-authored-by: Cory Martin --- parm/atm/obs/config/satwind_ahi_h8.yaml | 397 +++++++++++++++ .../bufr2ioda/bufr2ioda_satwind_amv_ahi.json | 15 + .../bufr2ioda/bufr2ioda_satwind_amv_ahi.py | 468 ++++++++++++++++++ 3 files changed, 880 insertions(+) create mode 100644 parm/atm/obs/config/satwind_ahi_h8.yaml create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.json create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.py diff --git a/parm/atm/obs/config/satwind_ahi_h8.yaml b/parm/atm/obs/config/satwind_ahi_h8.yaml new file mode 100644 index 000000000..c66afaa67 --- /dev/null +++ b/parm/atm/obs/config/satwind_ahi_h8.yaml @@ -0,0 +1,397 @@ +obs space: + name: satwind_ahi_h8 + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)satwnd.ahi_h8.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_satwind_ahi_h8_{{ current_cycle | to_YMDH }}.nc4 + io pool: + max pool size: 1 + simulated variables: [windEastward, windNorthward] + +obs operator: + name: VertInterp + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +linear obs operator: + name: VertInterp + +# NOTE: Tests using the Gaussian Thinning filter (below) to duplicate GSI's thinning of AHI/Himawari-8 satwinds +# results in more JEDI satwinds in the diag file than in GSI, but far fewer JEDI satwinds assimilated than +# GSI. JEDI under-counts assimilated winds by roughly 25-40%, relative to GSI, and this under-count is not +# even including the temporal thinning which is applied in GSI but not JEDI (by this filter below). See +# GDASApp Issue #741 for details: https://github.com/NOAA-EMC/GDASApp/issues/741 +#obs pre filters: +#- filter: Gaussian Thinning +# horizontal_mesh: 200 +# vertical_mesh: 10000 +# use_reduced_horizontal_grid: true +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + +obs prior filters: +# Apply variable changes needed for wind scaling +# For wind observations with pressure provided +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +obs post filters: +# Assign the initial observation error, based on height/pressure +# Hard-wiring to prepobs_errtable.global by Type +# ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML +# Type 242 (Himawari VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 242 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + + +# Type 250 (Himawari AHI WV, cloud-top or clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 250 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,7.,7.3,7.6,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.] + + +# Type Type 252 (Himawari AHI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 252 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# sanity-check criteria +# Observation Range Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -130. + maxvalue: 130. + action: + name: reject + +# Velocity Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Velocity + maxvalue: 130. + action: + name: reject + +# GSI read routine QC (part-1) +# Exclude Type 250 with windComputationMethod==5 (clear-sky WV) --- obs tossed without passing to setup routine +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_in: 250 + - variable: MetaData/windComputationMethod + is_in: 5 + action: + name: reject + +# Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/satelliteZenithAngle + maxvalue: 68. + action: + name: reject + +# GSI read routine QC (part-2) +# Reject obs with qualityInformationWithoutForecast < 85. (also > 100., which is missing data) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 85. + maxvalue: 100. + action: + name: reject + +# Reject Type 252 (IR) winds with a /=0 surface type (non-water surface) when latitude > 20. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: ObsType/windEastward + is_in: 252 + test variables: + - name: MetaData/latitude + maxvalue: 20. + action: + name: reject + +# GSI setupw routine QC +# Reject any ob Type [240–260] when pressure greater than 950 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240-260 + test variables: + - name: MetaData/pressure + maxvalue: 95001. + action: + name: reject + +# IR (Type 242), reject when pressure is less than 700 mb +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 242 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# cloud-top WV (Type 250), reject when pressure greater than 399 mb. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 250 + test variables: + - name: MetaData/pressure + maxvalue: 39900. + action: + name: reject + +# JMA IR (252) reject when pressure between 499 and 801 mb. +# PERFORM ACTION: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 49901. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 252 + action: + name: reject + +# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +# Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation +# Turn this check off for now. +# Need to check if troposphere pressure was implemented correctly in fv3-jed +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/tropopause_pressure + value: MetaData/pressure + minvalue: -5000. # 50 hPa above tropopause level, negative p-diff + action: + name: reject + +# All satwinds must adjust errors based on ObsErrorFactorPressureCheck +# prior to the SPDB check (i.e. the gross-error check). The gross-error +# check uses the adjusted errors for error-bound tightening and rejection, +# so this check has to come first. This check will inflate errors for obs +# that are too close to either the model top or bottom. +# Notes (eliu): GMAO added a required parameter: adjusted_error_name. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + surface_obs: false + variable: windEastward + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All satwinds subject to a gross-error check that contains significant +# modifiers for satwinds with a negative speed-bias. ALL wind gross-error +# checks are currently being done by the SatWindsSPDBCheck. +# CLEARED +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +# Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). +# : Turn off duplicate check for now. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# We are extending this to an additional filter that inflates final ob-errors across-the-board by +# 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), +# you would want to remove this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +# End of Filters diff --git a/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.json b/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.json new file mode 100644 index 000000000..964175368 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.json @@ -0,0 +1,15 @@ +{ + "data_format" : "bufr_d", + "data_type" : "satwnd", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : [ "NC005044", "NC005045", "NC005046" ], + "data_description" : "NC005044 JMA SATWIND, AHI HIM8 IR(LW)/VIS/WV-CT/WV-CS; NC005045 JMA SATWIND, HIMAWARI-8 IR(LW)/VIS/WV-CT/WV-CS; NC005046 JMA SATWIND, HIMAWARI-8 IR(LW)/VIS/WV-CT/WV-CS", + "data_provider" : "JMA", + "sensor_info" : { "sensor_name": "AHI", "sensor_full_name": "Advanced Himawari Imager", "sensor_id": 999 }, + "satellite_info" : [ + { "satellite_name": "H8", "satellite_full_name": "Himawari-8", "satellite_id": 173, "launch time": "YYYYMMDD" }, + ] +} diff --git a/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.py b/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.py new file mode 100755 index 000000000..8380b12aa --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.py @@ -0,0 +1,468 @@ +#!/usr/bin/env python3 +import argparse +import numpy as np +import numpy.ma as ma +from pyiodaconv import bufr +import calendar +import json +import time +import math +import datetime +import os +from datetime import datetime +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger + +# ==================================================================== +# Satellite Winds (AMV) BUFR dump file for AHI/Himawari +# ==================================================================== +# All subsets contain all spectral bands: NC005044 NC005045 NC005046 +# ==================================================================== +# Spectral Band | Code (002023) | ObsType +# -------------------------------------------------------------------- +# IRLW (Freq < 5E+13) | Method 1 | 252 +# VIS | Method 2 | 242 +# WV Cloud Top | Method 3 | 250 +# WV Clear Sky/ Deep Layer | Method 5 | 250 +# ==================================================================== + +# Define and initialize global variables +global float32_fill_value +global int32_fill_value +global int64_fill_value + +float32_fill_value = np.float32(0) +int32_fill_value = np.int32(0) +int64_fill_value = np.int64(0) + + +def Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd): + + uob = (-wspd * np.sin(np.radians(wdir))).astype(np.float32) + vob = (-wspd * np.cos(np.radians(wdir))).astype(np.float32) + + return uob, vob + + +def Get_ObsType(swcm, chanfreq): + + obstype = swcm.copy() + + # Use numpy vectorized operations + obstype = np.where(swcm == 5, 250, obstype) # WVCA/DL + obstype = np.where(swcm == 3, 250, obstype) # WVCT + obstype = np.where(swcm == 2, 242, obstype) # VIS + obstype = np.where(swcm == 1, 252, obstype) # IRLW + + if not np.any(np.isin(obstype, [242, 250, 252])): + raise ValueError("Error: Unassigned ObsType found ... ") + + return obstype + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + subsets = config["subsets"] + data_format = config["data_format"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + satellite_info_array = config["satellite_info"] + sensor_name = config["sensor_info"]["sensor_name"] + sensor_full_name = config["sensor_info"]["sensor_full_name"] + sensor_id = config["sensor_info"]["sensor_id"] + + # Get derived parameters + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + reference_time = datetime.strptime(cycle, "%Y%m%d%H") + reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") + + # General informaton + converter = 'BUFR to IODA Converter' + process_level = 'Level-2' + platform_description = 'Himawari-8' + sensor_description = 'Advanced Himawari Imager' + + logger.info(f'sensor_name = {sensor_name}') + logger.info(f'sensor_full_name = {sensor_full_name}') + logger.info(f'sensor_id = {sensor_id}') + logger.info(f'reference_time = {reference_time}') + + bufrfile = f"{cycle_type}.t{hh}z.{data_type}.tm00.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), 'atmos', bufrfile) + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + logger.info('Making QuerySet') + q = bufr.QuerySet(subsets) + + # MetaData + q.add('latitude', '*/CLAT') + q.add('longitude', '*/CLON') + q.add('satelliteId', '*/SAID') + q.add('year', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('second', '*/SECO') + q.add('satelliteZenithAngle', '*/SAZA') + q.add('sensorCentralFrequency', '*/SCCF') + q.add('pressure', '*/PRLC') + + # Processing Center + q.add('dataProviderOrigin', '*/OGCE') + q.add('windGeneratingApplication', '*/QCPRMS[1]/GNAP') + +# # Quality Infomation (Quality Indicator w/o forecast) + q.add('qualityInformationWithoutForecast', '*/QCPRMS[1]/PCCF') + + # Wind Retrieval Method Information + q.add('windComputationMethod', '*/SWCM') + + # ObsValue + q.add('windDirection', '*/WDIR') + q.add('windSpeed', '*/WSPD') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f'Processing time for making QuerySet : {running_time} seconds') + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + logger.info('Executing QuerySet to get ResultSet') + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + # MetaData + satid = r.get('satelliteId') + year = r.get('year') + month = r.get('month') + day = r.get('day') + hour = r.get('hour') + minute = r.get('minute') + second = r.get('second') + lat = r.get('latitude') + lon = r.get('longitude') + satzenang = r.get('satelliteZenithAngle') + pressure = r.get('pressure', type='float') + chanfreq = r.get('sensorCentralFrequency', type='float') + + # Processing Center + ogce = r.get('dataProviderOrigin') + ga = r.get('windGeneratingApplication') + + # Quality Information + qi = r.get('qualityInformationWithoutForecast', type='float') + # For AHI/Himawari data, qi w/o forecast (qifn) is packaged in same + # vector where ga == 102. Must conduct a search and extract the + # correct vector for gnap and qi + # 1. Find dimension-sizes of ga and qi (should be the same!) + gDim1, gDim2 = np.shape(ga) + qDim1, qDim2 = np.shape(qi) + logger.info(f'Generating Application and Quality Information SEARCH:') + logger.info(f'Dimension size of GNAP ({gDim1},{gDim2})') + logger.info(f'Dimension size of PCCF ({qDim1},{qDim2})') + # 2. Initialize gnap and qifn as None, and search for dimension of + # ga with values of 102. If the same column exists for qi, assign + # gnap to ga[:,i] and qifn to qi[:,i], else raise warning that no + # appropriate GNAP/PCCF combination was found + gnap = None + qifn = None + for i in range(gDim2): + if np.unique(ga[:, i].squeeze()) == 102: + if i <= qDim2: + logger.info(f'GNAP/PCCF found for column {i}') + gnap = ga[:, i].squeeze() + qifn = qi[:, i].squeeze() + else: + logger.info(f'ERROR: GNAP column {i} outside of PCCF dimension {qDim2}') + if (gnap is None) & (qifn is None): + logger.info(f'ERROR: GNAP == 102 NOT FOUND OR OUT OF PCCF DIMENSION-RANGE, WILL FAIL!') + + # Wind Retrieval Method Information + swcm = r.get('windComputationMethod') + + # ObsValue + # Wind direction and Speed + wdir = r.get('windDirection', type='float') + wspd = r.get('windSpeed') + + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + timestamp = r.get_datetime('year', 'month', 'day', 'hour', 'minute', 'second').astype(np.int64) + + # Check BUFR variable generic dimension and type + + # Global variables declaration + # Set global fill values + float32_fill_value = satzenang.fill_value + int32_fill_value = satid.fill_value + int64_fill_value = timestamp.fill_value.astype(np.int64) + + end_time = time.time() + running_time = end_time - start_time + logger.info(f'Processing time for executing QuerySet to get ResultSet : {running_time} seconds') + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + logger.info('Creating derived variables') + logger.debug('Creating derived variables - wind components (uob and vob)') + + uob, vob = Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd) + + logger.debug(f' uob min/max = {uob.min()} {uob.max()}') + logger.debug(f' vob min/max = {vob.min()} {vob.max()}') + + obstype = Get_ObsType(swcm, chanfreq) + + height = np.full_like(pressure, fill_value=pressure.fill_value, dtype=np.float32) + stnelev = np.full_like(pressure, fill_value=pressure.fill_value, dtype=np.float32) + + end_time = time.time() + running_time = end_time - start_time + logger.info(f'Processing time for creating derived variables : {running_time} seconds') + + # ===================================== + # Split output based on satellite id + # Create IODA ObsSpace + # Write IODA output + # ===================================== + logger.info('Create IODA ObsSpace and Write IODA output based on satellite ID') + + # Find unique satellite identifiers in data to process + unique_satids = np.unique(satid) + logger.info(f'Number of Unique satellite identifiers: {len(unique_satids)}') + logger.info(f'Unique satellite identifiers: {unique_satids}') + + logger.debug(f'Loop through unique satellite identifier {unique_satids}') + total_ob_processed = 0 + for sat in unique_satids.tolist(): + start_time = time.time() + + matched = False + for satellite_info in satellite_info_array: + if (satellite_info["satellite_id"] == sat): + matched = True + satellite_id = satellite_info["satellite_id"] + satellite_name = satellite_info["satellite_name"] + satinst = sensor_name.lower()+'_'+satellite_name.lower() + logger.debug(f'Split data for {satinst} satid = {sat}') + + if matched: + + # Define a boolean mask to subset data from the original data object + mask = satid == sat + # MetaData + lon2 = lon[mask] + lat2 = lat[mask] + timestamp2 = timestamp[mask] + satid2 = satid[mask] + satzenang2 = satzenang[mask] + chanfreq2 = chanfreq[mask] + obstype2 = obstype[mask] + pressure2 = pressure[mask] + height2 = height[mask] + stnelev2 = stnelev[mask] + + # Processing Center + ogce2 = ogce[mask] + + # QC Info + qifn2 = qifn[mask] + + # Method + swcm2 = swcm[mask] + + # ObsValue + wdir2 = wdir[mask] + wspd2 = wspd[mask] + uob2 = uob[mask] + vob2 = vob[mask] + + # Timestamp Range + timestamp2_min = datetime.fromtimestamp(timestamp2.min()) + timestamp2_max = datetime.fromtimestamp(timestamp2.max()) + + # Check unique observation time + unique_timestamp2 = np.unique(timestamp2) + logger.debug(f'Processing output for satid {sat}') + + # Create the dimensions + dims = { + 'Location': np.arange(0, wdir2.shape[0]) + } + + # Create IODA ObsSpace + iodafile = f"{cycle_type}.t{hh}z.{data_type}.{satinst}.tm00.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.info(f'Create output file : {OUTPUT_PATH}') + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug('Write global attributes') + obsspace.write_attr('Converter', converter) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeReference', reference_time) + obsspace.write_attr('datetimeRange', [str(timestamp2_min), str(timestamp2_max)]) + obsspace.write_attr('sensor', sensor_id) + obsspace.write_attr('platform', satellite_id) + obsspace.write_attr('platformCommonName', satellite_name) + obsspace.write_attr('sensorCommonName', sensor_name) + obsspace.write_attr('processingLevel', process_level) + obsspace.write_attr('platformLongDescription', platform_description) + obsspace.write_attr('sensorLongDescription', sensor_description) + + # Create IODA variables + logger.debug('Write variables: name, type, units, and attributes') + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon2.dtype, fillval=lon2.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon2) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat2.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat2) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=np.int64, fillval=int64_fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(timestamp2) + + # Satellite Identifier + obsspace.create_var('MetaData/satelliteIdentifier', dtype=satid2.dtype, fillval=satid2.fill_value) \ + .write_attr('long_name', 'Satellite Identifier') \ + .write_data(satid2) + + # Sensor Zenith Angle + obsspace.create_var('MetaData/satelliteZenithAngle', dtype=satzenang2.dtype, fillval=satzenang2.fill_value) \ + .write_attr('units', 'degree') \ + .write_attr('valid_range', np.array([0, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Satellite Zenith Angle') \ + .write_data(satzenang2) + + # Sensor Centrall Frequency + obsspace.create_var('MetaData/sensorCentralFrequency', dtype=chanfreq2.dtype, fillval=chanfreq2.fill_value) \ + .write_attr('units', 'Hz') \ + .write_attr('long_name', 'Satellite Channel Center Frequency') \ + .write_data(chanfreq2) + + # Data Provider + obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce2.dtype, fillval=ogce2.fill_value) \ + .write_attr('long_name', 'Identification of Originating/Generating Center') \ + .write_data(ogce2) + + # Quality: Percent Confidence - Quality Information Without Forecast + obsspace.create_var('MetaData/qualityInformationWithoutForecast', dtype=qifn2.dtype, fillval=qifn2.fill_value) \ + .write_attr('long_name', 'Quality Information Without Forecast') \ + .write_data(qifn2) + + # Wind Computation Method + obsspace.create_var('MetaData/windComputationMethod', dtype=swcm2.dtype, fillval=swcm2.fill_value) \ + .write_attr('long_name', 'Satellite-derived Wind Computation Method') \ + .write_data(swcm2) + + # ObsType based on computation method/spectral band + obsspace.create_var('ObsType/windEastward', dtype=obstype2.dtype, fillval=swcm2.fill_value) \ + .write_attr('long_name', 'Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') \ + .write_data(obstype2) + + # ObsType based on computation method/spectral band + obsspace.create_var('ObsType/windNorthward', dtype=obstype2.dtype, fillval=swcm2.fill_value) \ + .write_attr('long_name', 'Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') \ + .write_data(obstype2) + + # Pressure + obsspace.create_var('MetaData/pressure', dtype=pressure2.dtype, fillval=pressure2.fill_value) \ + .write_attr('units', 'pa') \ + .write_attr('long_name', 'Pressure') \ + .write_data(pressure2) + + # Height (mimic prepbufr) + obsspace.create_var('MetaData/height', dtype=height2.dtype, fillval=height2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Height of Observation') \ + .write_data(height2) + + # Station Elevation (mimic prepbufr) + obsspace.create_var('MetaData/stationElevation', dtype=stnelev2.dtype, fillval=stnelev2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Station Elevation') \ + .write_data(stnelev2) + + # U-Wind Component + obsspace.create_var('ObsValue/windEastward', dtype=uob2.dtype, fillval=wspd2.fill_value) \ + .write_attr('units', 'm s-1') \ + .write_attr('long_name', 'Eastward Wind Component') \ + .write_data(uob2) + + # V-Wind Component + obsspace.create_var('ObsValue/windNorthward', dtype=vob2.dtype, fillval=wspd2.fill_value) \ + .write_attr('units', 'm s-1') \ + .write_attr('long_name', 'Northward Wind Component') \ + .write_data(vob2) + + end_time = time.time() + running_time = end_time - start_time + total_ob_processed += len(satid2) + logger.debug(f'Number of observation processed : {len(satid2)}') + logger.debug(f'Processing time for splitting and output IODA for {satinst} : {running_time} seconds') + + else: + logger.info(f"Do not find this satellite id in the configuration: satid = {sat}") + + logger.info("All Done!") + logger.info(f'Total number of observation processed : {total_ob_processed}') + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('BUFR2IODA_satwind_amv_ahi.py', level=log_level, colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.info(f"Total running time: {running_time} seconds") From 673c086c09058adbd0b2898225ceaa289d04ae9d Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Fri, 17 Nov 2023 15:45:27 -0600 Subject: [PATCH 13/58] amsua converter initial set --- .../bufr2ioda/amsua_metop-c_v2.ACCoeff.nc | Bin 0 -> 20109 bytes .../bufr2ioda_combine_ncep_1bamua_ta.yml | 190 ++++++++++++++++++ .../bufr2ioda_combine_ncep_esamua.yml | 187 +++++++++++++++++ .../bufr2ioda/bufr2ioda_ncep_1bamua_ta.json | 18 ++ parm/ioda/bufr2ioda/bufr2ioda_ncep_amsua.json | 19 ++ ush/ioda/bufr2ioda/antcorr_application.py | 40 ++++ ush/ioda/bufr2ioda/bufr2ioda_base.py | 69 +++++++ .../bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 94 +++++++++ ush/ioda/bufr2ioda/run_bufr2ioda.py | 78 +++++-- ush/ioda/bufr2ioda/utils.py | 87 ++++++++ 10 files changed, 769 insertions(+), 13 deletions(-) create mode 100644 parm/ioda/bufr2ioda/amsua_metop-c_v2.ACCoeff.nc create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_1bamua_ta.yml create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_esamua.yml create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.json create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_ncep_amsua.json create mode 100644 ush/ioda/bufr2ioda/antcorr_application.py create mode 100644 ush/ioda/bufr2ioda/bufr2ioda_base.py create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py create mode 100644 ush/ioda/bufr2ioda/utils.py diff --git a/parm/ioda/bufr2ioda/amsua_metop-c_v2.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_metop-c_v2.ACCoeff.nc new file mode 100644 index 0000000000000000000000000000000000000000..c7a31156e6b9f19068a59dd128087f4a1e964e54 GIT binary patch literal 20109 zcmeHt2V7IzvUeyVB4RHfLQueh(!l~a^j_v!?uK)|%N_?r3Y@PeDb2 zMqKjpG+EjJ(O2rvck){4F7Bc3spp~8mp^Ge9BmoW{kBOtbgSi5Jau_c>CW=dS8S9$p>d%ejg`51;X`pMMuWLY4G$xjiq74--l9m{i%xA~O z@ls;=?8vC3ROa*lIfQG!@-wG(wi)qM2cdd`>d1&*pKE0!P4OCr8Ed z#WP3!G}BPuP+w2qK+nivBHh4vnz8;gBNO^0eY{vq8%1*(O$JHI(-dhVXbgS~pOeI= z4UiWvPNUJ05HcYpW`knp=a!%#E1}>c*2#a4f(eb*3n?gw#Pa3~lHwBCPCU`dVsZKr zJdpRC*rXH=JC>gu7e6UX(UpewlcEh2vG42V!S+UF#KaIfl~DM*ndyZ~DB99!y^*5Q z4~joA^!F-enlz0zQhs2a7^BX?b&*mA5zE6Br|WLPQ)} zTPHU=cW);Tceb~Um8%`;FrlQw2yg^(iQUL-jQH-ExS=6>{^DxyEj;UCw3?LfjAnxm&r}5zh&MEb&7L#6*Xp<9`kvJ=Dh5 zp3XxL5fDyG=cXjn1#!tg9;I>;qd43czAl}-jPz+~H`?F-u?hR*D#S&#|5X#B^N21= zF>-&Zn195Cnq-Xo|4nEoj-RwHl83)GVK150Hwev(4V2$oJc$h9KMW*CgdUodEM)=( z0ox{$BM|Uol4?mHZ9IGL+5eCY4NPbm7b1m!rwtVe;QZ|3y+l2P`;n~EWJL~1-BAvy zp()nJ@2&Y44rwY<6B+Yj;sg5`HKpien<3RHWDfuSpA3h^LTlX+r`s zgzQD<`03i68`Aj0hG+YTA30DXC?}KCAbO-^F^GvSB{C+H>#rrz)YfoSD6*qH!KoAm zOu`p={G_nNsCZ&$5=|y#Ap#vGi!1JN^tf<(k|?aw!^GgA3sPdaxOXUy5pQwgK$Rkh zN>0ML1Py{YiiW$M6%nV2jjOLHPP6T!Vq$zaF)5@5e>Nh{lSE#M0M!!D_Ox&4k^p3! zXtX%oN}QXD)0DGQ_oUO)fz+}k&A2bk_&ahcHc8Y~+4}iSr zEAIb~08t?xNXb%w>21Xp(LX}UmZhy!FG2<$vi=$w8U!8jDl{uWGG8Fz(D~uvQDISh zLD(#Mcw8b~M46r%#ZUYD!Qn}PV+aPd--Bb|9~ev|&!3!f#O!szt(6YS= z{|U4h5i_W>_#cH8A|^--F)lualZ;LpO9Cd81VW2=U25b>bJ;?|%yP;i>J%Diassx! zhfh*pGLBGZ2lDVoQ?O!F^c9E{0I%>RdzD3i@gZpc)t;k^qa}R)ab!uNas@iI%)bI~ z*5x?4iA2O8MkH4K@r*v=r?a`MlQ)w_BQ!~z-G}2x6zTsYco8btL2g{c_c7=1nmE{a zxQbLm5WtI46SXr@2Q_N{|9w9d_J7|`*_G<2>`K{_>ZjjwV_KodcO2%php8e8tBnLj|O@)(4&DK4fJTBM*}?? z=+Qur2LA78fT%9?8bGA|#H$S|njTewKo$bgXQbIr9&S@Gm<3&oukbzz0zS`dPC??Wc6_VsTaYh4_a7 z{b|Wq0b|(N+cE6iZS2_M3fM4&8&SZ*pFiMl7Lxy_E~!eHNhEVrZ&6~QXN}AODjc6kf)%_YMAN*I8nyUR#EhQykL*k;Or0yvxsrz6v zooM1a5i1B9g!YK~Ab|tI_JlQvm_ft|qSQ>hfgozXL}8gI*S#$E^@~3%1eYT2+btJ_ zP*@ndqT;d;R$WaSpHeA=WeuTEG;4%#jjfTyzafON56+F+cUuUu$?V#gdLdMeP!HF8 zAOucx_RiXeLil)q))d<)gxCZA5p$b_(6axqaK&RGSdN(a#sZ(`WmvR#;Jh0h;r$x1 z-5$-h)WiOO&Y1!8aNHPIFWr?mPdZnAWIocl*0&_v{gDuI(tTyW<2qALI;Yj&6GG=p zRhh#$ze(Vv)KDjc>dqA9ZscoE-RiQ6TSC}axFUYtO(EPJl@-gtwmY}bt3S3gjNEOQ zPlS+k@5R<_%|ck0#O>eSA_TK5Lx(%G;(D4ddu@9r1XH=Z0n=Ux;n)LRAtA1 zTPsWUEknK3w946xx{LZUFz-6jfa{l;MIXm;OPqr$%8(8!<$4FM^EztjS?!j*LLsa?%4}TKf#c5Bx@~!jxT$tLy|+^cj^|!C5A|R; z#W^Go<*j}&(7i9>VM%_?{_ALO3(oT1>k*fu<4$}zREhdtrchUdxILOZ^qc7=A@sd* zEH~u}>QQwT{qR*GsPDWv>Qb!`6^H{21zK-MRta>SATqARXDkuIgj_l+Z@7vOs1 z`k5#=p+1uI^mT?H9}d%G#^|A5RPFAnI3u6O@-qceaILoDXwW#<& zwA-8k(w$n!r(wyemZeofc;PX!uI-!@cAFqTPYR*YyiQ~CMZ_PixhMqX zFFIbsTvj23_@gbEO{gEoUmI>tK{{!J=iYvg@@B8JEH6Tw6jr1kNWt|E_j|Z-8uCHm z{X}>0wy7wG;nY$iSL8ou{-oj{w9|;vudD^w9%|e-bPu+@gM6>aW7})U%eFCSPnC0l zbSuQeu#Dg#7I;5?`g9F5e15C7=1Bn33EMnx`AnpHdxqn~Fx02N{#ysH=V(u^PGkV` zv3YxD90&23F0*c=9pcm_@m!Jib;NZ={~Y^kLXaCUeSIX_nOB@Pza0I(L~3Aj70T1f zuGXnXeNp&l8Pb>uXum}V2TfE$`bFckWDlU;bMEws9F2CEyWmarX5?qVsn_*+h&u|0 z?m8Kky2!sSv%9n$aT*!G&d z{Pc*byAt8OOWm;`lekK zxZc87E0mw0oI|ePleR`a_8uPhdMEOARr~Y6vuNM8+{(!hFwR?E&=0Mw6T%-Gp`*Ce1xM1?`=*^}&)kh~rO}rm@bTeqZP=KQbEaeqr~gPfJkmvpqdm_@KR2 zJ3R$%{*yxZu3h}V7RDoydZ%+Z74c_`no*i}he@P52@+=9(G z-lG4T+WRQyG}%?MDo8hPTl16-T&K<6k9i4kJ%ZRMkq<|&h&LJ-m+!psl{$lbI&X^l z>WjEGIP|&96!}-KTAn)emJo7}&FBA$xYm$X+TmOygaWs%4}~`n?{~I%y+b|Cai87v z3GE-uD19W0e(U3W{rw=6*H>#z`f6M!?!AHf0MzrfsLNAvox^3{uO{QVWiw}9Ux<91p7OSC723J^`PtsZh-Woo z+eI9^-;U@VggBlzl3V@^<>`^P$G$yy?(yG$mw1&dIxynCn17_#5{%yuj9jGeVEi5( zuX!UB&(%?dzvZ>x#C?54t;scv+h5btpA}vg!pXsSLdJMLt2khD2ku+ngN%9RRYKU( z{n}y0RowUV45iNE`Sb`o?c_T=uNur#`S|QO<}LdgRT}o=KDMXrh)fZlhjF)6IfZ#v z_be5CoHrwr=~a$&`YP!yb42>Nnj6-i!u2qT!jl5#3Egu8OOU^%=f4d##JIcK<7({# zjKjq@6SptLeWdHgbe|5~k8rcn48%CC=$AM&3-^^vufFs;h5OZ)>$&RlqY~hNwalTI zXYuf@;O#9pZagGkKWSgRPylb8*FOI;JqB$2KYa<aL|5Y zt#*DT4-Om&2+dy31HH_|oxvGAxOPW#ri}_8BFEIbjW6cIn`^QcPh>_xN-A-}kA?3V zzjdWNj)CStzsZfGqTo`_ku>{2K788CTz>dC4=4PxWbm*=D6R7y1$Gj0gpG?^K84C1=GsxPb`>uVwvz#p)YJQdNA}$s6XVL$uIWT4TMo?&b?oK4}z8{9WzcC zg~2R!ha_HW1YFKGNSmY)2lBL47ISoyAboaB^@1bGu(s_++pNi{Kpe9e^Adq6%^jrs zGyxvXcI&U;oB)%KCgt|J9}C+OU59;CkAltPtY4;f@ku$j;Az^bAWY=In$Z25ONWF) z^R?ny*M}X< zf(E;rGU|(2px5-Jq;Ve$MmiK;-b?cX`QqUF@qPjDp^mjMKrsXoyS0amejEyVf?ZPO zy}0oDk-Ctj69y@TjM;lOqJOQK^|Zg70PYdx81*E${VBtDt$7->m=wP(95xf)B{rm2 zbj&34xI`$*eZFk&#zeR}rfkh&yF`c&y5iQ)I1W4$RA-hxi~=GXl&a^$K9d?V_nRE} zb--1vRpy~^PFC=!#yc1o_4mIrhXz1PfrHA!yFQ>jt$L#MAr?ISX!_f|Y$m)n4qg2| ziUrGUckmy9k2vlyeI@rNR$Pu#&|yK-qpCr9d>>dCQkKNo?F-x?ve|LM0I;d@*q=Tv z1U9T~)m(oj6e@i-dP{S;Bu>I%Zr+`Zzvj)5#GePB=glZjPJ)n5_pVkp``P$lLHqn9*uib90Em7$F&mA`;&D5CKM_wKh?d! zglQrJGoev!$%46kSb)L}S;Jw$`S>DE<`r+q=v`AZM~4C1+O)11{>FeZtgf`xdPCLF z#@5kiyy0$z4T{KsC$kC~Hr0E0a3IR7Qdw}?$2!evD+~1Vt!%!_`ogYO z%l@UYelVe>DI#b}C@k_;cWj)>1DfBcskf9PAw6j1ge#w;p&heS^S3b&>`gy%{Rsu$;GQdI_m1WRgE<#(>+t;{_{+xK zhL?lj-nw(oRL#S{S`@e=LCti{=DtA!NY9!3TGulXlr4_m$dOMb<4g+NnhKW_9;ZN- z7Xio=u#JlA$f!&PtS$-iU&ccXv5hFt0E5^1+P|IV!GygQTF)TT zFSg$(@q=vv=_#!3KAEU?Jmpk7wx4Ry1a-E~12qb-aAEf^Wi(^w2+)tzklIxp4ON~)z1zmdz-Z6k7CUCf^vK&|-yS^o`0u|> zylQ=&chg6U1O5+&WM|FbKuu+q>$zeM6e%qzJK)QKZ0}^vjo(Ab_f&Xpq-#_SKN*RxGSn22absWjLpTBbUGW)bpm=kf7<-}vdq0Fp@ zx9fr-KfTpz_KF}_kqmq35C|j7WN@nvgo9rztdr$~z-XoR@UYNeFm{bCSX>?gEm>VN zHAaQN+Uog^XVZhhf9b9R$!W2$@Ou9-OB|vhTsMAJp>h;?PLF`Ax`uq=xhGRh>VWd+s$qInW1Em%lu?@E`+( z1zWq;Z}o=ut>Ft(monkX*MzUF>saJ@gbAAmBstZkdc*81XLv2=eL$qnfdOEW8}!S< zlpyF|Rm#H3T7z+2F6j!Qz2!&T__kv8WbKq;BbcaU)7o^{R?XJlR6W7P% zd2oIEj`@qS_|TB=KWO`&aCkB7ZQ+P~yY!Zt#KEeRFb@Gdwr$d(>>VBivxW z7`W@XJsj?=4*kW^7P21C*SUAd3Ov&CXVwn01Z~yXSxY~f!xXqz2m1$h&ij0Wwn0_WQ07Duz~ z;8TLJgU>uCfbLs6r&qdx&E+Zc2P=3%?9-FuTT2;|yox$+AH>(6UVJ}T{LpUG&Y%FO z3zP}e@(Y5H6OE?c8X=^dY}i&!B<2A?jX&w6D6O;9N0QI%EZtd?An=AGmZxl1NHM@E z|JB#{)t-{`jx!S+$UNT;#Pe(*?rjHJ#EQ)GY(Wfsdr(=%H(I*e z9yZEY+&RXvCH=t~&g?b_aA4U0TTA^^-}SZ-*!Red+9-QckB&f8vyF6J!B4gJ+YBX7 zXgoOarhoX4^9+@jPPIv>*z(N>RON`1l0VQYlV7|q4}|VbclR7R9S){yF^3*R@FjHx z!m7hgav^Mi?ci}-4s7(XX;yP#ll7fIm}H~+s@XIEVtSA1>a*PscqPruXpWC0f0Z&R zjX9{tAoD0s(l5P0t1>BFU5P>F{R}vzO}x4Dgop3GMeUmA1|xzSRg3D}p}6#8!31MZ z(*9m>Wrd3C#SSm>9QDIL5;U?O+gJHO=ca{KpFaA-COlQE$_B%uH?7mBDuhC3uye5C zD$M_hv}I5jgw*Rb^m`BvowYj@Sq6N_yh0=pVIaLFslzprOP)J9a5b=KUd&oH+WZ?{gOA9aUCqrUfAM00|HqD0glw(pDa z^Y*uu*jpUutfB5&fP&*!fR2w@_tvtkNWZrx>jXA%L6105+kyOaIUV=XANHPL`Yh#= zh6|Y|yTdWJw`Kc=d69JrZ_+O;5bDGqq(0+zS}<_L zt3b$To73-nfds-kRj8_xNQc3Sfv_LVnO275+-tDf642x2B|%YsyA-gk##U9xb057 zp?8BXp~@pq-t&MpCk*$QPVoZj+%AqcOj1uQGB0BR{Z<^7j{HIE^#do{Prj0R3l&Gj z?{yjQW$US6JvT3MfAApVfh%Yq82ymPbRzL=PsUjrGEcCARfh(LEb<1pOoaFKmT*E@ zaPIk1D=7Ugb$5TTjl_;tWzq^C-dn+A9imWY3k$X-Jh61NhgrnyA$cd#k6a|{Q+b6L zr{0phg37B#H~o4?tHlp0bOu3*guYAk5A(|%m<$RO)s?%)@9N8$EI zXE-x-d7l}de#EELFIK_3J?+W7%8uNhZD5IGrfT4IOW1o|33nbVxb^i)eZIf7WM2^1 zpX?#=ab04Bwj;TJJCpw9D#^q9jLZ)iU+oF|w`84d@52CH`cbtru1uJ8>yp{Ibv{sN zzOcl*a0XnG>pM28I})}KWgSKY%vhdt^XhXxOx4+`{beo>4jEkb7EZ70@4rpFy3G92C*pWG9Ibq^&RrAhvSaI} zkIIgM)Jqy89fwClZ~WO!;nFG-B#faPgr6Sfu`{vQ;$PW}J@ literal 0 HcmV?d00001 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_1bamua_ta.yml b/parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_1bamua_ta.yml new file mode 100644 index 000000000..345d4bd65 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_1bamua_ta.yml @@ -0,0 +1,190 @@ +# (C) Copyright 2021 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "{{ DMPDIR }}/{{ RUN }}.{{ PDY }}/{{ cyc }}/atmos/{{ RUN }}.t{{ cyc }}z.1bamua.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + + latitude: + query: "*/CLAT" + + longitude: + query: "*/CLON" + + satelliteIdentifier: + query: "*/SAID" + + satelliteInstrument: + query: "*/SIID" + + fieldOfViewNumber: + query: "*/FOVN" + + landOrSeaQualifier: + query: "*/LSQL" + + heightOfLandSurface: + query: "*/HOLS" + + heightOfStation: + query: "*/HMSL" + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + sensorScanAngle: + sensorScanAngle: + fieldOfViewNumber: "*/FOVN" + scanStart: -48.333 + scanStep: 3.333 + sensor: amsua + + sensorChannelNumber: + query: "*/BRITCSTC/CHNM" + + # ObsValue + # Note: BUFR Dump contains Antenna Temperature for all normal-feed AMSUAs except NOAA-15/16 + # NOAA-15/16 contain Brightness Temperature + antennaTemperature: + query: "*/BRITCSTC/TMBR" + + splits: + satId: + category: + variable: satelliteIdentifier + map: + _3: metop-b + _4: metop-a + _5: metop-c + _208: n17 + _209: n18 + _223: n19 + + ioda: + backend: netcdf + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.amsua.$(splitvar)_ta.tm00.nc" + + dimensions: + - name: Channel + source: variables/sensorChannelNumber + path: "*/BRITCSTC" + + globals: + - name: "platformCommonName" + type: string + value: "AMSU-A" + + - name: "platformLongDescription" + type: string + value: "MTYP 021-023 PROC AMSU-A 1B Tb" + + variables: + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteIdentifier + longName: "Satellite Identifier" + + - name: "MetaData/satelliteInstrument" + source: variables/satelliteInstrument + longName: "Satellite Instrument" + + - name: "MetaData/landOrSeaQualifier" + source: variables/landOrSeaQualifier + longName: "Land/Sea Qualifier" + + - name: "MetaData/heightOfLandSurface" + source: variables/heightOfLandSurface + longName: "Height of Land Surface" + units: "m" + + - name: "MetaData/heightOfStation" + source: variables/heightOfStation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degree" + range: [0, 180] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [0, 90] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorViewAngle" + source: variables/sensorScanAngle + longName: "Sensor View Angle" + units: "degree" + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + - name: "MetaData/sensorScanPosition" + source: variables/fieldOfViewNumber + longName: "Field of View Number" + + # ObsValue + # For now, store the antennaTemperature as brightnessTemperature + - name: "ObsValue/brightnessTemperature" + source: variables/antennaTemperature + longName: "Antenna Temperature" + units: "K" + range: [90, 380] + chunks: [1000, 15] + compressionLevel: 4 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_esamua.yml b/parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_esamua.yml new file mode 100644 index 000000000..d79c3a20e --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_esamua.yml @@ -0,0 +1,187 @@ +# (C) Copyright 2021 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "{{ DMPDIR }}/{{ RUN }}.{{ PDY }}/{{ cyc }}/atmos/{{ RUN }}.t{{ cyc }}z.esamua.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + + latitude: + query: "*/CLATH" + + longitude: + query: "*/CLONH" + + satelliteIdentifier: + query: "*/SAID" + + satelliteAntennaCorrectionsVersionNumber: + query: "*/SACV" + + orbitNumber: + query: "*/ORBN" + + scanLineNumber: + query: "*/SLNM" + + fieldOfViewNumber: + query: "*/FOVN" + + heightOfStation: + query: "*/SELV" + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + sensorScanAngle: + sensorScanAngle: + fieldOfViewNumber: "*/FOVN" + scanStart: -48.333 + scanStep: 3.333 + sensor: amsua + + sensorChannelNumber: + query: "*/ATNCHV/INCN" + + # ObsValue + brightnessTemperature: + query: "*/ATNCHV/TMBRST" + + splits: + satId: + category: + variable: satelliteIdentifier + map: + _3: metop-b + _4: metop-a + _5: metop-c +# _206: n15 + _209: n18 + _223: n19 + + ioda: + backend: netcdf + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.esamua.$(splitvar).tm00.nc" + + dimensions: + - name: Channel + source: variables/sensorChannelNumber + path: "*/ATNCHV" + + globals: + - name: "platformCommonName" + type: string + value: "AMSUA" + + - name: "platformLongDescription" + type: string + value: "MTYP 021-033 RARS(EARS,AP,SA) AMSU-A 1C Tb DATA)" + + variables: + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteIdentifier + longName: "Satellite Identifier" + + - name: "MetaData/satelliteAntennaCorrectionsVersionNumber" + source: variables/satelliteAntennaCorrectionsVersionNumber + longName: "Satellite Antenna Corrections Version Number" + + - name: "MetaData/fieldOfViewNumber" + source: variables/fieldOfViewNumber + longName: "Field Of View Number" + + - name: "MetaData/orbitNumber" + source: variables/orbitNumber + longName: "Orbit Number" + + - name: "MetaData/scanLineNumber" + source: variables/scanLineNumber + longName: "Scan Line Number1" + + - name: "MetaData/heightOfStation" + source: variables/heightOfStation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degree" + range: [0, 180] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [0, 90] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorViewAngle" + source: variables/sensorScanAngle + longName: "Sensor View Angle" + units: "degree" + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + # ObsValue + - name: "ObsValue/brightnessTemperature" + coordinates: "longitude latitude Channel" + source: variables/brightnessTemperature + longName: "Brightness Temperature" + units: "K" + range: [100, 500] + chunks: [1000, 15] + compressionLevel: 4 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.json b/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.json new file mode 100644 index 000000000..70c56246d --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.json @@ -0,0 +1,18 @@ +{ + "data_format" : "bufr_d", + "data_type" : "1bamua_td", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : [ "NC005030", "NC005031", "NC005032", "NC005034", "NC005039" ], + "data_description" : "NC005030 NESDIS SATWIND, GOES IR(LW); NC005031 NESDIS SATWIND, GOES WV-IMG/DL; NC005032 NESDIS SATWIND, GOES VIS; NC005034 NESDIS SATWIND, GOES WV-IMG/CT; NC005039 NESDIS SATWIND, GOES IR(SW)", + "data_provider" : "U.S. NOAA/NDESDIS", + "sensor_info" : { "sensor_name": "ABI", "sensor_full_name": "Advanced Baseline Imager", "sensor_id": 617 }, + "satellite_info" : [ + { "satellite_name": "GOES-16", "satellite_full_name": "Geostationary Operational Satellite - 16", "satellite_id": 270 }, + { "satellite_name": "GOES-17", "satellite_full_name": "Geostationary Operational Satellite - 17", "satellite_id": 271 }, + { "satellite_name": "GOES-18", "satellite_full_name": "Geostationary Operational Satellite - 18", "satellite_id": 272 } + ], + "yaml_file" : "bufr_ncep_1bamua_ta.yaml" +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ncep_amsua.json b/parm/ioda/bufr2ioda/bufr2ioda_ncep_amsua.json new file mode 100644 index 000000000..5f6153b42 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_ncep_amsua.json @@ -0,0 +1,19 @@ +{ + "data_format" : "bufr_d", + "data_type" : ["esamua", "1bamua_td"], + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : [ "NC005030", "NC005031", "NC005032", "NC005034", "NC005039" ], + "data_description" : "NC005030 NESDIS SATWIND, GOES IR(LW); NC005031 NESDIS SATWIND, GOES WV-IMG/DL; NC005032 NESDIS SATWIND, GOES VIS; NC005034 NESDIS SATWIND, GOES WV-IMG/CT; NC005039 NESDIS SATWIND, GOES IR(SW)", + "data_provider" : "U.S. NOAA/NDESDIS", + "sensor_info" : { "sensor_name": "ABI", "sensor_full_name": "Advanced Baseline Imager", "sensor_id": 617 }, + "satellite_info" : [ + { "satellite_name": "GOES-16", "satellite_full_name": "Geostationary Operational Satellite - 16", "satellite_id": 270 }, + { "satellite_name": "GOES-17", "satellite_full_name": "Geostationary Operational Satellite - 17", "satellite_id": 271 }, + { "satellite_name": "GOES-18", "satellite_full_name": "Geostationary Operational Satellite - 18", "satellite_id": 272 } + ], + "yaml_file" : ["{{ COM_OBS }}/ncep_esamua_{{ current_cycle | to_YMDH }}.yaml", + "{{ COM_OBS }}/ncep_1bamua_ta_{{ current_cycle | to_YMDH }}.yaml"] +} diff --git a/ush/ioda/bufr2ioda/antcorr_application.py b/ush/ioda/bufr2ioda/antcorr_application.py new file mode 100644 index 000000000..187af1728 --- /dev/null +++ b/ush/ioda/bufr2ioda/antcorr_application.py @@ -0,0 +1,40 @@ +import netCDF4 as nc +import numpy as np + +INVALID = -1.0 +# Cosmic background temperature. Taken from Mather,J.C. et. al., 1999, "Calibrator Design for the COBE +# Far-Infrared Absolute Spectrophotometer (FIRAS)"Astrophysical Journal, vol 512, pp 511-520 +TSPACE = 2.7253 + + +class ACCoeff: + def __init__(self, sat_id='metop-c'): + file_name = 'amsua_' + sat_id + '_v2.ACCoeff.nc' + nc_file = nc.Dataset(file_name) + self.n_FOVS = len(nc_file.dimensions['n_FOVs']) + self.n_Channels = len(nc_file.dimensions['n_Channels']) + self.A_earth = nc_file.variables['A_earth'][:] + self.A_platform = nc_file.variables['A_platform'][:] + self.A_space = nc_file.variables['A_space'][:] + + +def remove_ant_corr(AC, iFOV, T): + # AC: Structure containing the antenna correction coefficients for the sensor of interest. + # iFOV: The FOV index for a scanline of the sensor of interest. + # T: On input, this argument contains the brightness + + if iFOV < 1 or iFOV > AC.n_FOVS or len(T) != AC.n_Channels: + return [INVALID] * len(T) + iFOV = iFOV - 1 + T = AC.A_earth[:, iFOV] * T + AC.A_platform[:, iFOV] * T + AC.A_space[:, iFOV] * TSPACE + return T + + +def apply_ant_corr(AC, iFOV, T): + # T: On input, this argument contains the antenna temperatures for the sensor channels. + if iFOV < 1 or iFOV > AC.n_FOVS or len(T) != AC.n_Channels: + return [INVALID] * len(T) + iFOV = iFOV - 1 + T = (T - AC.A_space[:, iFOV] * TSPACE) / (AC.A_earth[:, iFOV] + AC.A_platform[:, iFOV]) + return T + diff --git a/ush/ioda/bufr2ioda/bufr2ioda_base.py b/ush/ioda/bufr2ioda/bufr2ioda_base.py new file mode 100644 index 000000000..45cb5ab18 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_base.py @@ -0,0 +1,69 @@ +import json +import re +import yaml + +from pyiodaconv import bufr + +#from wxflow import Logger +from logging import Logger + +from utils import timing_decorator + +logger = Logger('BUFR2IODA_satwind_amv_goes.py', level='DEBUG') + + +class Bufr2IodaBase: + + def __init__(self, file_name): + with open(file_name, "r") as json_file: + config = json.load(json_file) + self.config = config + yaml_config = self.get_yaml_config(config) + self.yaml_config = yaml_config + self.ioda_files = yaml_config['observations'][0]['ioda'].get('obsdataout') + logger.info(f'Ioda output files are: {self.ioda_files}') + self.splits = yaml_config['observations'][0]['obs space']['exports'].get('splits') + self.split_files = None + + def check_yaml_config(yaml_config): + # TODO make some consistency check? + pass + + def get_attr(self, config): + # TODO add more attr into the ioda file from config + obs_attr = {} + return obs_attr + + def get_data_container(self): + pass + + def get_yaml_file(self): + return self.config['yaml_file'] + + def get_yaml_config(self, config): + # TODO add and/or modify ymal contents from config json file. e.g. file names, etc. + # What does en_bufr2ioda_yaml.py do? connection to this? + yaml_file = self.get_yaml_file() + with open(yaml_file, 'r') as file: + yaml_config = yaml.load(file, Loader=yaml.FullLoader) + return yaml_config + + def make_split_files(self, sat_ids): + # sat_id a list of ids + pattern = r'{(.*?)}' + self. split_files = [re.sub(pattern, sat_id[0], self.ioda_files) for sat_id in sat_ids] + + def re_map_variable(self, container, ioda_description): + # Make any changes and return for your specific case in the sub-class + pass + + @timing_decorator + def execute(self): + yaml_path = self.get_yaml_file() + container = bufr.Parser(yaml_path).parse() + ioda_description = bufr.IodaDescription(yaml_path) + if self.splits: + self.make_split_files(container.allSubCategories()) + self.re_map_variable(container, ioda_description) + bufr.IodaEncoder(ioda_description).encode(container) + diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py new file mode 100755 index 000000000..619d38ec4 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 + +import argparse +import os +from bufr2ioda_base import Bufr2IodaBase +#from wxflow import Logger +from logging import Logger +from antcorr_application import ACCoeff, apply_ant_corr +from utils import timing_decorator, nc_merge + +logger = Logger('BUFR2IODA_satwind_amv_goes.py', level='DEBUG') + +R1000 = 1000.0 +R1000000 = 1000000.0 + +SPLIT = ('metop-a', 'metop-b', 'metop-c', 'n17', 'n18', 'n19') +ioda_files = [(f'esamua.{x}.tm00.nc', f'amsua.{x}_ta.tm00.nc', f'amsua.{x}.tm00.nc') for x in SPLIT] + + +class Bufr2IodaAmusa(Bufr2IodaBase): + def get_yaml_file(self): + return self.config['yaml_file'][0] + + +class Bufr2IodaEbmua(Bufr2IodaBase): + + def get_yaml_file(self): + return self.config['yaml_file'][1] + + @timing_decorator + def re_map_variable(self, container, ioda_description): + # TODO replace this follow that in GSI + # read_bufrtovs.f90 + # antcorr_application.f90 + # search the keyword “ta2tb” for details + + sat_ids = container.allSubCategories() + for sat_id in sat_ids: + ta = container.get('variables/antennaTemperature', sat_id) + if ta.shape[0]: + ifov = container.get('variables/fieldOfViewNumber', sat_id) + tb = self.apply_ant_corr(sat_id, ta, ifov) + container.replace('variables/antennaTemperature', tb, sat_id) + + def apply_ant_corr(self, sat_id, ta, ifov): + ac = ACCoeff() # TODO add later + llll = 1 # TODO how to set this + if llll == 1: + if sat_id not in ['n15', 'n16']: + # Convert antenna temperature to brightness temperature + for i in range(ta.shape[0]): + x = ta[i, :] + apply_ant_corr(ac, ifov[i], x) # TODO if this is too slow we might need to optimize it + x[x > R1000] = R1000000 + else: + pass # TODO after know how to set llll + return ta + +@timing_decorator +def merge(amsua_files): + amsua_0 = amsua_files[0] + amsua_1 = amsua_files[1] + obs_Path_prefix = os.path.commonprefix([amsua_files[0][0], amsua_files[1][0]]) + for ioda_file_0, ioda_file_1, ioda_file_t in ioda_files: + try: + nc_merge(obs_Path_prefix + ioda_file_0, obs_Path_prefix + ioda_file_1, obs_Path_prefix + ioda_file_t) + except FileNotFoundError as e: + print(e) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + #parser.add_argument('-t', '--type', type=str, help='Input Satellite type a/e', required=True) + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('BUFR2IODA_ncep_amsua.py', level=log_level) + amsua_files = [] + for sat_type in ['a', 'e']: + print(sat_type) + if sat_type == 'a': + convert = Bufr2IodaAmusa(args.config) + else: + convert = Bufr2IodaEbmua(args.config) + convert.execute() + amsua_files.append(convert.split_files) + print(amsua_files) + logger.info('Converting amsua type {} done!'.format(sat_type)) + + print('--start to merge--') + merge(amsua_files) + print('--Finished merge--') diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 1130658b1..d44c29b21 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -1,14 +1,37 @@ #!/usr/bin/env python3 import argparse +import glob +import multiprocessing as mp import os +import shutil +from itertools import repeat from pathlib import Path from gen_bufr2ioda_json import gen_bufr_json +from gen_bufr2ioda_yaml import gen_bufr_yaml from wxflow import (Logger, Executable, cast_as_dtype, logit, to_datetime, datetime_to_YMDH, Task, rm_p) # Initialize root logger logger = Logger('run_bufr2ioda.py', level='INFO', colored_log=True) +# get parallel processing info +num_cores = mp.cpu_count() + + +def mp_bufr_py(script, infile): + cmd = Executable(script) + cmd.add_default_arg('-c') + cmd.add_default_arg(infile) + logger.info(f"Executing {cmd}") + cmd() + + +def mp_bufr_yaml(bufr2iodaexe, yamlfile): + cmd = Executable(bufr2iodaexe) + cmd.add_default_arg(yamlfile) + logger.info(f"Executing {cmd}") + cmd() + @logit(logger) def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): @@ -17,6 +40,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # Get gdasapp root directory DIR_ROOT = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../..")) USH_IODA = os.path.join(DIR_ROOT, "ush", "ioda", "bufr2ioda") + BIN_GDAS = os.path.join(DIR_ROOT, "build", "bin") # Create output directory if it doesn't exist os.makedirs(COM_OBS, exist_ok=True) @@ -26,14 +50,39 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): 'RUN': RUN, 'current_cycle': current_cycle, 'DMPDIR': DMPDIR, - 'COM_OBS': COM_OBS + 'COM_OBS': COM_OBS, + 'PDY': current_cycle.strftime('%Y%m%d'), + 'cyc': current_cycle.strftime('%H'), } - # Specify observation types to be processed by a script + # copy necessary fix files to runtime directory + fix_files = ["atms_beamwidth.txt", + "amsua_metop-c_v2.ACCoeff.nc" + ] + for fix_file in fix_files: + shutil.copy(os.path.join(config_template_dir, fix_file), os.path.join(os.getcwd(), fix_file)) + + script_type = 'bufr2ioda_combine_' + # Specify observation types to be processed by the bufr2ioda_ executable + BUFR_yml_files = glob.glob(os.path.join(config_template_dir, '*.yml')) + BUFR_yml_files = [os.path.basename(f) for f in BUFR_yml_files] + BUFR_yml = [f.replace(script_type, '').replace('.yml', '') for f in BUFR_yml_files] - BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr", "adpsfc_prepbufr", "sfcshp_prepbufr"] - BUFR_py = ["ncep_amsua",] - for obtype in BUFR_py: + for obtype in BUFR_yml: + logger.info(f"Convert {obtype}...") + yaml_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.yaml") + filename = script_type + obtype + '.yml' + template = os.path.join(config_template_dir, filename) + gen_bufr_yaml(config, template, yaml_output_file) + +# Specify observation types to be processed by a script with combined methods + BUFR_combine_files = glob.glob(os.path.join(USH_IODA, script_type + '*.py')) + BUFR_combine_files = [os.path.basename(f) for f in BUFR_combine_files] + BUFR_combine = [f.replace(script_type, '').replace('.py', '') for f in BUFR_combine_files] + + json_files = [] + scripts = [] + for obtype in BUFR_combine: logger.info(f"Convert {obtype}...") json_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") filename = 'bufr2ioda_' + obtype + '.json' @@ -41,16 +90,19 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): gen_bufr_json(config, template, json_output_file) # Use the converter script for the ob type - bufr2iodapy = USH_IODA + '/bufr2ioda_' + obtype + ".py" - cmd = Executable(bufr2iodapy) - cmd.add_default_arg('-c') - cmd.add_default_arg(json_output_file) - logger.info(f"Executing {cmd}") - cmd() + bufr2iodapy = os.path.join(USH_IODA, script_type + obtype + ".py") + + # append the values to the lists + json_files.append(json_output_file) + scripts.append(bufr2iodapy) # Check if the converter was successful - if os.path.exists(json_output_file): - rm_p(json_output_file) + # if os.path.exists(json_output_file): + # rm_p(json_output_file) + + # run all python scripts in parallel + with mp.Pool(num_cores) as pool: + pool.starmap(mp_bufr_py, zip(scripts, json_files)) if __name__ == "__main__": diff --git a/ush/ioda/bufr2ioda/utils.py b/ush/ioda/bufr2ioda/utils.py new file mode 100644 index 000000000..b1f7c62e7 --- /dev/null +++ b/ush/ioda/bufr2ioda/utils.py @@ -0,0 +1,87 @@ +import netCDF4 as nc +import numpy as np +import time + + +def nc_merge(file_name1, file_name2, target_name): + ncf1 = nc.Dataset(file_name1) + ncf2 = nc.Dataset(file_name2) + ncf = nc.Dataset(target_name, "w", format="NETCDF4") + + # Add attr for dataset + ncf1_attr = ncf1.__dict__ + ncf2_attr = ncf2.__dict__ + ncf_attr = {} + for k, v1 in ncf1_attr.items(): + ncf_attr[k] = v1 + v2 = ncf2_attr.get(k) + if v2 and v1 != v2: + ncf_attr[k] = f"1: {v1}, 2: {v2}" + ncf.setncatts(ncf_attr) + + # Create groups + for grp in ncf1.groups: + print(grp) + ncf.createGroup(grp) + + # Create dimensions + ncf1_dims = ncf1.dimensions + ncf2_dims = ncf2.dimensions + for dim_key in ncf1_dims: + dim1 = ncf1_dims[dim_key] + dim2 = ncf2_dims.get(dim_key) + if dim2: + if dim1.name == 'Location': + dim_size = dim1.size + dim2.size + else: + dim_size = dim1.size + ncf.createDimension(dim1.name, dim_size) + else: + print('error: dimension mismatch') # TODO throw an exception + + # Create Dimension variables + vars2 = ncf2.variables + for var1 in ncf1.variables.values(): + if var1.name == 'Channel': + var = var1 + else: + var = np.concatenate((var1, vars2[var1.name])) + ncf_var = ncf.createVariable(var1.name, var1.datatype, var1.dimensions) + ncf_var[:] = var[:] + attrs = var1.ncattrs() + for attr in attrs: + ncf_var.setncattr(attr, var1.getncattr(attr)) + + # Create Group variables + grp1 = ncf1.groups + grp2 = ncf2.groups + for k in grp1.keys(): + vars1 = grp1[k].variables + vars2 = grp2[k].variables + for key in vars1: + var = vars1[key] + if vars2.get(key): + var2 = vars2[key] + else: + print(f"-----------Warning: {key} is not existed in ncf2, skip -----------------") + continue + var_array = np.concatenate((var, var2)) + var_path = f"{grp1[k].path}/{var.name}" + var_fill_value = var.getncattr('_FillValue') + ncf_var = ncf.createVariable(var_path, var.datatype, var.dimensions, fill_value=var_fill_value) + ncf_var[:] = var_array[:] + attrs = var.ncattrs() + for attr in attrs: + if attr != '_FillValue': + ncf_var.setncattr(attr, var.getncattr(attr)) + + +def timing_decorator(func): + def wrapper(*args, **kwargs): + start_time = time.time() + result = func(*args, **kwargs) + end_time = time.time() + execution_time = end_time - start_time + print(f"Function '{func.__name__}' took {execution_time:.6f} seconds to execute.") + return result + return wrapper From 820d8451fc23679759b9315e898f823b88b50077 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Sun, 19 Nov 2023 07:23:30 -0600 Subject: [PATCH 14/58] forgot undating run_bufr2ioda.py --- ush/ioda/bufr2ioda/run_bufr2ioda.py | 85 +++++++++++++++-------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index b15b96f42..a0852b07d 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -56,38 +56,12 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): } # copy necessary fix files to runtime directory -<<<<<<< HEAD fix_files = ["atms_beamwidth.txt", "amsua_metop-c_v2.ACCoeff.nc" ] for fix_file in fix_files: shutil.copy(os.path.join(config_template_dir, fix_file), os.path.join(os.getcwd(), fix_file)) - script_type = 'bufr2ioda_combine_' - # Specify observation types to be processed by the bufr2ioda_ executable - BUFR_yml_files = glob.glob(os.path.join(config_template_dir, '*.yml')) - BUFR_yml_files = [os.path.basename(f) for f in BUFR_yml_files] - BUFR_yml = [f.replace(script_type, '').replace('.yml', '') for f in BUFR_yml_files] - - for obtype in BUFR_yml: - logger.info(f"Convert {obtype}...") - yaml_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.yaml") - filename = script_type + obtype + '.yml' - template = os.path.join(config_template_dir, filename) - gen_bufr_yaml(config, template, yaml_output_file) - -# Specify observation types to be processed by a script with combined methods - BUFR_combine_files = glob.glob(os.path.join(USH_IODA, script_type + '*.py')) - BUFR_combine_files = [os.path.basename(f) for f in BUFR_combine_files] - BUFR_combine = [f.replace(script_type, '').replace('.py', '') for f in BUFR_combine_files] - - json_files = [] - scripts = [] - for obtype in BUFR_combine: -======= - shutil.copy(os.path.join(config_template_dir, "atms_beamwidth.txt"), - os.path.join(os.getcwd(), "atms_beamwidth.txt")) - # Specify observation types to be processed by a script BUFR_py_files = glob.glob(os.path.join(USH_IODA, 'bufr2ioda_*.py')) BUFR_py_files = [os.path.basename(f) for f in BUFR_py_files] @@ -96,7 +70,6 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): json_files = [] scripts = [] for obtype in BUFR_py: ->>>>>>> feature/gdas-validation logger.info(f"Convert {obtype}...") json_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") filename = 'bufr2ioda_' + obtype + '.json' @@ -104,25 +77,11 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): gen_bufr_json(config, template, json_output_file) # Use the converter script for the ob type -<<<<<<< HEAD - bufr2iodapy = os.path.join(USH_IODA, script_type + obtype + ".py") -======= bufr2iodapy = USH_IODA + '/bufr2ioda_' + obtype + ".py" ->>>>>>> feature/gdas-validation # append the values to the lists json_files.append(json_output_file) scripts.append(bufr2iodapy) -<<<<<<< HEAD - - # Check if the converter was successful - # if os.path.exists(json_output_file): - # rm_p(json_output_file) - - # run all python scripts in parallel - with mp.Pool(num_cores) as pool: - pool.starmap(mp_bufr_py, zip(scripts, json_files)) -======= # Check if the converter was successful # if os.path.exists(json_output_file): @@ -158,7 +117,49 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # run all bufr2ioda yamls in parallel with mp.Pool(num_cores) as pool: pool.starmap(mp_bufr_yaml, zip(repeat(bufr2iodaexe), yaml_files)) ->>>>>>> feature/gdas-validation + + script_type = 'bufr2ioda_combine_' + # Specify observation types to be processed by the bufr2ioda_ executable + BUFR_yml_files = glob.glob(os.path.join(config_template_dir, '*.yml')) + BUFR_yml_files = [os.path.basename(f) for f in BUFR_yml_files] + BUFR_yml = [f.replace(script_type, '').replace('.yml', '') for f in BUFR_yml_files] + + for obtype in BUFR_yml: + logger.info(f"Convert {obtype}...") + yaml_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.yaml") + filename = script_type + obtype + '.yml' + template = os.path.join(config_template_dir, filename) + gen_bufr_yaml(config, template, yaml_output_file) + + +# Specify observation types to be processed by a script with combined methods + BUFR_combine_files = glob.glob(os.path.join(USH_IODA, script_type + '*.py')) + BUFR_combine_files = [os.path.basename(f) for f in BUFR_combine_files] + BUFR_combine = [f.replace(script_type, '').replace('.py', '') for f in BUFR_combine_files] + + json_files = [] + scripts = [] + for obtype in BUFR_combine: + logger.info(f"Convert {obtype}...") + json_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") + filename = 'bufr2ioda_' + obtype + '.json' + template = os.path.join(config_template_dir, filename) + gen_bufr_json(config, template, json_output_file) + + # Use the converter script for the ob type + bufr2iodapy = os.path.join(USH_IODA, script_type + obtype + ".py") + + # append the values to the lists + json_files.append(json_output_file) + scripts.append(bufr2iodapy) + + # Check if the converter was successful + # if os.path.exists(json_output_file): + # rm_p(json_output_file) + + # run all python scripts in parallel + with mp.Pool(num_cores) as pool: + pool.starmap(mp_bufr_py, zip(scripts, json_files)) if __name__ == "__main__": From e4bc8e0f050c1f9186d540f9c0f4ed5474bb30ea Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Sun, 19 Nov 2023 07:50:41 -0600 Subject: [PATCH 15/58] a working version --- ush/ioda/bufr2ioda/run_bufr2ioda.py | 70 +---------------------------- 1 file changed, 1 insertion(+), 69 deletions(-) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index a0852b07d..f22509951 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -62,77 +62,9 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): for fix_file in fix_files: shutil.copy(os.path.join(config_template_dir, fix_file), os.path.join(os.getcwd(), fix_file)) - # Specify observation types to be processed by a script - BUFR_py_files = glob.glob(os.path.join(USH_IODA, 'bufr2ioda_*.py')) - BUFR_py_files = [os.path.basename(f) for f in BUFR_py_files] - BUFR_py = [f.replace('bufr2ioda_', '').replace('.py', '') for f in BUFR_py_files] - - json_files = [] - scripts = [] - for obtype in BUFR_py: - logger.info(f"Convert {obtype}...") - json_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") - filename = 'bufr2ioda_' + obtype + '.json' - template = os.path.join(config_template_dir, filename) - gen_bufr_json(config, template, json_output_file) - - # Use the converter script for the ob type - bufr2iodapy = USH_IODA + '/bufr2ioda_' + obtype + ".py" - - # append the values to the lists - json_files.append(json_output_file) - scripts.append(bufr2iodapy) - - # Check if the converter was successful - # if os.path.exists(json_output_file): - # rm_p(json_output_file) - - # run all python scripts in parallel - with mp.Pool(num_cores) as pool: - pool.starmap(mp_bufr_py, zip(scripts, json_files)) - - # Specify observation types to be processed by the bufr2ioda executable - BUFR_yaml_files = glob.glob(os.path.join(config_template_dir, '*.yaml')) - BUFR_yaml_files = [os.path.basename(f) for f in BUFR_yaml_files] - BUFR_yaml = [f.replace('bufr2ioda_', '').replace('.yaml', '') for f in BUFR_yaml_files] - - yaml_files = [] - for obtype in BUFR_yaml: - logger.info(f"Convert {obtype}...") - yaml_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.yaml") - filename = 'bufr2ioda_' + obtype + '.yaml' - template = os.path.join(config_template_dir, filename) - gen_bufr_yaml(config, template, yaml_output_file) - - # use the bufr2ioda executable for the ob type - bufr2iodaexe = BIN_GDAS + '/bufr2ioda.x' - - # append the values to the lists - yaml_files.append(yaml_output_file) - - # Check if the converter was successful - # if os.path.exists(yaml_output_file): - # rm_p(yaml_output_file) - - # run all bufr2ioda yamls in parallel - with mp.Pool(num_cores) as pool: - pool.starmap(mp_bufr_yaml, zip(repeat(bufr2iodaexe), yaml_files)) - - script_type = 'bufr2ioda_combine_' - # Specify observation types to be processed by the bufr2ioda_ executable - BUFR_yml_files = glob.glob(os.path.join(config_template_dir, '*.yml')) - BUFR_yml_files = [os.path.basename(f) for f in BUFR_yml_files] - BUFR_yml = [f.replace(script_type, '').replace('.yml', '') for f in BUFR_yml_files] - - for obtype in BUFR_yml: - logger.info(f"Convert {obtype}...") - yaml_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.yaml") - filename = script_type + obtype + '.yml' - template = os.path.join(config_template_dir, filename) - gen_bufr_yaml(config, template, yaml_output_file) - # Specify observation types to be processed by a script with combined methods + script_type = 'bufr2ioda_combine_' BUFR_combine_files = glob.glob(os.path.join(USH_IODA, script_type + '*.py')) BUFR_combine_files = [os.path.basename(f) for f in BUFR_combine_files] BUFR_combine = [f.replace(script_type, '').replace('.py', '') for f in BUFR_combine_files] From 10f01dbfe379a053b05f1c44139a56e544c1a4ab Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 20 Nov 2023 08:36:02 -0500 Subject: [PATCH 16/58] Write the bufr2ioda config files in DATA (#748) Instead of writing the filled out `.json` and `.yaml` files to `$COM_OBS`, we write them to `$DATA` so that they get purged unless `KEEPDATA="YES"` --- ush/ioda/bufr2ioda/run_bufr2ioda.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 45dad6539..153cf2de5 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -36,6 +36,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): DIR_ROOT = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../..")) USH_IODA = os.path.join(DIR_ROOT, "ush", "ioda", "bufr2ioda") BIN_GDAS = os.path.join(DIR_ROOT, "build", "bin") + DATA = os.getcwd() # Create output directory if it doesn't exist os.makedirs(COM_OBS, exist_ok=True) @@ -52,7 +53,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # copy necessary fix files to runtime directory shutil.copy(os.path.join(config_template_dir, "atms_beamwidth.txt"), - os.path.join(os.getcwd(), "atms_beamwidth.txt")) + os.path.join(DATA, "atms_beamwidth.txt")) # Specify observation types to be processed by a script BUFR_py_files = glob.glob(os.path.join(USH_IODA, 'bufr2ioda_*.py')) @@ -63,7 +64,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): exename = [] for obtype in BUFR_py: logger.info(f"Convert {obtype}...") - json_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") + json_output_file = os.path.join(DATA, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") filename = 'bufr2ioda_' + obtype + '.json' template = os.path.join(config_template_dir, filename) gen_bufr_json(config, template, json_output_file) @@ -86,7 +87,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): for obtype in BUFR_yaml: logger.info(f"Convert {obtype}...") - yaml_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.yaml") + yaml_output_file = os.path.join(DATA, f"{obtype}_{datetime_to_YMDH(current_cycle)}.yaml") filename = 'bufr2ioda_' + obtype + '.yaml' template = os.path.join(config_template_dir, filename) gen_bufr_yaml(config, template, yaml_output_file) From bcfefae9159f1130e2f54b26f347ee5445a1e3b6 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Mon, 20 Nov 2023 12:00:49 -0600 Subject: [PATCH 17/58] remove a json file --- .../bufr2ioda/bufr2ioda_ncep_1bamua_ta.json | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.json diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.json b/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.json deleted file mode 100644 index 70c56246d..000000000 --- a/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "data_format" : "bufr_d", - "data_type" : "1bamua_td", - "cycle_type" : "{{ RUN }}", - "cycle_datetime" : "{{ current_cycle | to_YMDH }}", - "dump_directory" : "{{ DMPDIR }}", - "ioda_directory" : "{{ COM_OBS }}", - "subsets" : [ "NC005030", "NC005031", "NC005032", "NC005034", "NC005039" ], - "data_description" : "NC005030 NESDIS SATWIND, GOES IR(LW); NC005031 NESDIS SATWIND, GOES WV-IMG/DL; NC005032 NESDIS SATWIND, GOES VIS; NC005034 NESDIS SATWIND, GOES WV-IMG/CT; NC005039 NESDIS SATWIND, GOES IR(SW)", - "data_provider" : "U.S. NOAA/NDESDIS", - "sensor_info" : { "sensor_name": "ABI", "sensor_full_name": "Advanced Baseline Imager", "sensor_id": 617 }, - "satellite_info" : [ - { "satellite_name": "GOES-16", "satellite_full_name": "Geostationary Operational Satellite - 16", "satellite_id": 270 }, - { "satellite_name": "GOES-17", "satellite_full_name": "Geostationary Operational Satellite - 17", "satellite_id": 271 }, - { "satellite_name": "GOES-18", "satellite_full_name": "Geostationary Operational Satellite - 18", "satellite_id": 272 } - ], - "yaml_file" : "bufr_ncep_1bamua_ta.yaml" -} From 8762fbd259dd2d7f66581bbbe5175f9b02c11272 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Mon, 20 Nov 2023 13:35:32 -0600 Subject: [PATCH 18/58] correct code norm --- ush/ioda/bufr2ioda/antcorr_application.py | 3 +-- ush/ioda/bufr2ioda/bufr2ioda_base.py | 10 ++++------ ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 14 +++++++------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/ush/ioda/bufr2ioda/antcorr_application.py b/ush/ioda/bufr2ioda/antcorr_application.py index 187af1728..49eff1fca 100644 --- a/ush/ioda/bufr2ioda/antcorr_application.py +++ b/ush/ioda/bufr2ioda/antcorr_application.py @@ -36,5 +36,4 @@ def apply_ant_corr(AC, iFOV, T): return [INVALID] * len(T) iFOV = iFOV - 1 T = (T - AC.A_space[:, iFOV] * TSPACE) / (AC.A_earth[:, iFOV] + AC.A_platform[:, iFOV]) - return T - + return T \ No newline at end of file diff --git a/ush/ioda/bufr2ioda/bufr2ioda_base.py b/ush/ioda/bufr2ioda/bufr2ioda_base.py index 45cb5ab18..8bec8ae01 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_base.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_base.py @@ -4,8 +4,7 @@ from pyiodaconv import bufr -#from wxflow import Logger -from logging import Logger +from wxflow import Logger from utils import timing_decorator @@ -36,10 +35,10 @@ def get_attr(self, config): def get_data_container(self): pass - + def get_yaml_file(self): return self.config['yaml_file'] - + def get_yaml_config(self, config): # TODO add and/or modify ymal contents from config json file. e.g. file names, etc. # What does en_bufr2ioda_yaml.py do? connection to this? @@ -65,5 +64,4 @@ def execute(self): if self.splits: self.make_split_files(container.allSubCategories()) self.re_map_variable(container, ioda_description) - bufr.IodaEncoder(ioda_description).encode(container) - + bufr.IodaEncoder(ioda_description).encode(container) \ No newline at end of file diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index 619d38ec4..be32c928f 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -3,8 +3,7 @@ import argparse import os from bufr2ioda_base import Bufr2IodaBase -#from wxflow import Logger -from logging import Logger +from wxflow import Logger from antcorr_application import ACCoeff, apply_ant_corr from utils import timing_decorator, nc_merge @@ -20,7 +19,7 @@ class Bufr2IodaAmusa(Bufr2IodaBase): def get_yaml_file(self): return self.config['yaml_file'][0] - + class Bufr2IodaEbmua(Bufr2IodaBase): @@ -43,7 +42,7 @@ def re_map_variable(self, container, ioda_description): container.replace('variables/antennaTemperature', tb, sat_id) def apply_ant_corr(self, sat_id, ta, ifov): - ac = ACCoeff() # TODO add later + ac = ACCoeff() # TODO add later llll = 1 # TODO how to set this if llll == 1: if sat_id not in ['n15', 'n16']: @@ -56,6 +55,7 @@ def apply_ant_corr(self, sat_id, ta, ifov): pass # TODO after know how to set llll return ta + @timing_decorator def merge(amsua_files): amsua_0 = amsua_files[0] @@ -70,7 +70,7 @@ def merge(amsua_files): if __name__ == '__main__': parser = argparse.ArgumentParser() - #parser.add_argument('-t', '--type', type=str, help='Input Satellite type a/e', required=True) + # parser.add_argument('-t', '--type', type=str, help='Input Satellite type a/e', required=True) parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) parser.add_argument('-v', '--verbose', help='print debug logging information', action='store_true') @@ -82,7 +82,7 @@ def merge(amsua_files): print(sat_type) if sat_type == 'a': convert = Bufr2IodaAmusa(args.config) - else: + else: convert = Bufr2IodaEbmua(args.config) convert.execute() amsua_files.append(convert.split_files) @@ -91,4 +91,4 @@ def merge(amsua_files): print('--start to merge--') merge(amsua_files) - print('--Finished merge--') + print('--Finished merge--') \ No newline at end of file From 2d32978652990fd639fff855293f4981a6a3a39a Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Mon, 20 Nov 2023 13:44:14 -0600 Subject: [PATCH 19/58] a few more --- ush/ioda/bufr2ioda/antcorr_application.py | 3 ++- ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 3 ++- ush/ioda/bufr2ioda/run_bufr2ioda.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ush/ioda/bufr2ioda/antcorr_application.py b/ush/ioda/bufr2ioda/antcorr_application.py index 49eff1fca..187af1728 100644 --- a/ush/ioda/bufr2ioda/antcorr_application.py +++ b/ush/ioda/bufr2ioda/antcorr_application.py @@ -36,4 +36,5 @@ def apply_ant_corr(AC, iFOV, T): return [INVALID] * len(T) iFOV = iFOV - 1 T = (T - AC.A_space[:, iFOV] * TSPACE) / (AC.A_earth[:, iFOV] + AC.A_platform[:, iFOV]) - return T \ No newline at end of file + return T + diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index be32c928f..5ffe1e10e 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -91,4 +91,5 @@ def merge(amsua_files): print('--start to merge--') merge(amsua_files) - print('--Finished merge--') \ No newline at end of file + print('--Finished merge--') + diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 8bdf603f0..e2d27aea9 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -125,7 +125,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # append the values to the lists config_files.append(json_output_file) exename.append(bufr2iodapy) - + # run everything in parallel with mp.Pool(num_cores) as pool: pool.starmap(mp_bufr_converter, zip(exename, config_files)) From c6ebf2faaa8173bbd835a88083844d7090055fc5 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Mon, 20 Nov 2023 13:48:59 -0600 Subject: [PATCH 20/58] a few more --- ush/ioda/bufr2ioda/antcorr_application.py | 1 + ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 1 + ush/ioda/bufr2ioda/run_bufr2ioda.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/antcorr_application.py b/ush/ioda/bufr2ioda/antcorr_application.py index 187af1728..c773c99e1 100644 --- a/ush/ioda/bufr2ioda/antcorr_application.py +++ b/ush/ioda/bufr2ioda/antcorr_application.py @@ -38,3 +38,4 @@ def apply_ant_corr(AC, iFOV, T): T = (T - AC.A_space[:, iFOV] * TSPACE) / (AC.A_earth[:, iFOV] + AC.A_platform[:, iFOV]) return T + diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index 5ffe1e10e..bd1a85e88 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -93,3 +93,4 @@ def merge(amsua_files): merge(amsua_files) print('--Finished merge--') + diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index e2d27aea9..5d2d28a6f 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -125,7 +125,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # append the values to the lists config_files.append(json_output_file) exename.append(bufr2iodapy) - + # run everything in parallel with mp.Pool(num_cores) as pool: pool.starmap(mp_bufr_converter, zip(exename, config_files)) From 1a1d09e502fde7750d2ae2338d02457551828bc9 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Mon, 20 Nov 2023 13:57:04 -0600 Subject: [PATCH 21/58] a few more --- ush/ioda/bufr2ioda/antcorr_application.py | 2 -- ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 1 - ush/ioda/bufr2ioda/run_bufr2ioda.py | 1 + 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ush/ioda/bufr2ioda/antcorr_application.py b/ush/ioda/bufr2ioda/antcorr_application.py index c773c99e1..fb541c96a 100644 --- a/ush/ioda/bufr2ioda/antcorr_application.py +++ b/ush/ioda/bufr2ioda/antcorr_application.py @@ -37,5 +37,3 @@ def apply_ant_corr(AC, iFOV, T): iFOV = iFOV - 1 T = (T - AC.A_space[:, iFOV] * TSPACE) / (AC.A_earth[:, iFOV] + AC.A_platform[:, iFOV]) return T - - diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index bd1a85e88..5ffe1e10e 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -93,4 +93,3 @@ def merge(amsua_files): merge(amsua_files) print('--Finished merge--') - diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 5d2d28a6f..c1a6ab36c 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -140,3 +140,4 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): parser.add_argument('COM_OBS', type=Path, help='path to output ioda format dump files') args = parser.parse_args() bufr2ioda(args.current_cycle, args.RUN, args.DMPDIR, args.config_template_dir, args.COM_OBS) + From d9820944fe1588499098ce7297312da86e0316b1 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Mon, 20 Nov 2023 13:59:33 -0600 Subject: [PATCH 22/58] a few more --- ush/ioda/bufr2ioda/run_bufr2ioda.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index c1a6ab36c..386349ea4 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -125,7 +125,6 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # append the values to the lists config_files.append(json_output_file) exename.append(bufr2iodapy) - # run everything in parallel with mp.Pool(num_cores) as pool: pool.starmap(mp_bufr_converter, zip(exename, config_files)) From a910c8249344f260451d828a2f395d338cd5ee07 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Mon, 20 Nov 2023 14:05:19 -0600 Subject: [PATCH 23/58] dele last line --- ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 1 - ush/ioda/bufr2ioda/run_bufr2ioda.py | 1 - 2 files changed, 2 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index 5ffe1e10e..223452424 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -92,4 +92,3 @@ def merge(amsua_files): print('--start to merge--') merge(amsua_files) print('--Finished merge--') - diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 386349ea4..8a4a9debc 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -139,4 +139,3 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): parser.add_argument('COM_OBS', type=Path, help='path to output ioda format dump files') args = parser.parse_args() bufr2ioda(args.current_cycle, args.RUN, args.DMPDIR, args.config_template_dir, args.COM_OBS) - From 56810e2e654ae5d5d4a21bdca2585bde91307022 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Mon, 20 Nov 2023 14:30:07 -0600 Subject: [PATCH 24/58] no new line at the end of the file --- ush/ioda/bufr2ioda/bufr2ioda_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_base.py b/ush/ioda/bufr2ioda/bufr2ioda_base.py index 8bec8ae01..a26d7819f 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_base.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_base.py @@ -64,4 +64,4 @@ def execute(self): if self.splits: self.make_split_files(container.allSubCategories()) self.re_map_variable(container, ioda_description) - bufr.IodaEncoder(ioda_description).encode(container) \ No newline at end of file + bufr.IodaEncoder(ioda_description).encode(container) From 7845c79b54a8cd82b4faaa32e196d62f5d007767 Mon Sep 17 00:00:00 2001 From: Xuanli Li <101414760+XuanliLi-NOAA@users.noreply.github.com> Date: Mon, 20 Nov 2023 16:27:23 -0500 Subject: [PATCH 25/58] Update GNSSRO assimilation to end-to-end GDASApp validation (#753) Updating GNSSRO converter and adding the yaml file to GDASApp end-to-end testing based on [#712](https://github.com/NOAA-EMC/GDASApp/pull/712). New file: parm/atm/obs/config/gnssro.yaml: Added QC filters in the YAML Updated files: parm/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.json: Updated the satellite information ush/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.py: Updated pccf, qfro, and satelliteAscending flag Refer to [issue #750](https://github.com/NOAA-EMC/GDASApp/issues/750) for testing details. JEDI/GSI comparisons revealed a cutoff near obs=0.03 Rad due to super refraction 2. The number of observations after QC in JEDI is also inconsistent with GSI. --- parm/atm/obs/config/gnssro.yaml | 139 ++++ parm/atm/obs/lists/gdas_prototype_3d.yaml | 5 +- .../ioda/bufr2ioda/bufr2ioda_gnssro_bufr.json | 34 + ush/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.py | 650 ++++++++++++++++++ 4 files changed, 826 insertions(+), 2 deletions(-) create mode 100644 parm/atm/obs/config/gnssro.yaml create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.json create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.py diff --git a/parm/atm/obs/config/gnssro.yaml b/parm/atm/obs/config/gnssro.yaml new file mode 100644 index 000000000..949a4f87b --- /dev/null +++ b/parm/atm/obs/config/gnssro.yaml @@ -0,0 +1,139 @@ +obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + sr_steps: 2 + vertlayer: full + super_ref_qc: NBAM +obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)gnssro.tm00.bufr_d.nc +# obsgrouping: +# group variables: [ 'sequenceNumber' ] +# sort variable: 'impactHeightRO' +# sort order: 'ascending' + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_gnssro_{{ current_cycle | to_YMDH }}.nc4 + simulated variables: [bendingAngle] + +obs filters: +# Apply gross check using pccf +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for pccf +- filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: pccfCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for qfro +- filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: qfroCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: pccf Check - good: 0.1-100, reject: 0 +- filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265-269,750-755,44,786,820,825 + test variables: + - name: MetaData/pccf + minvalue: 0.1 + maxvalue: 100.1 + actions: + - name: set + flag: pccfCheckReject + - name: reject + +# Step 0-B: qfro Check - good: 0, reject: 1 +- filter: Bounds Check +- filter: RejectList + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 3-5,421,440,821 + test variables: + - name: MetaData/qualityFlags + minvalue: -0.1 + maxvalue: 0.1 + actions: + - name: set + flag: qfroCheckReject + - name: reject + +#1. gpstop +- filter: Domain Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/impactHeightRO + minvalue: 0 + maxvalue: 55000.1 + action: + name: reject +#2. commgpstop +- filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266,267,268,269 + test variables: + - name: MetaData/impactHeightRO + maxvalue: 45000.1 + action: + name: reject +#3. metop below 8 km +- filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 3-5 + test variables: + - name: MetaData/impactHeightRO + minvalue: 8000.1 + action: + name: reject +#4. assign obs error +- filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM +#5. RONBAM cut off check +- filter: Background Check RONBAM + filter variables: + - name: bendingAngle +#6. Obs error inflate +- filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate +#7. Background check +#- filter: Background Check +# filter variables: +# - name: bendingAngle +# threshold: 10 +# action: +# name: reject + diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml b/parm/atm/obs/lists/gdas_prototype_3d.yaml index ca478da42..98877f47c 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml @@ -3,11 +3,12 @@ observers: ##- !INC ${OBS_YAML_DIR}/sondes_prepbufr.yaml ##- !INC ${OBS_YAML_DIR}/atms_n20.yaml ##- !INC ${OBS_YAML_DIR}/aircraft.yaml -- !INC ${OBS_YAML_DIR}/satwind_goes-16.yaml +##- !INC ${OBS_YAML_DIR}/satwind_goes-16.yaml ##- !INC ${OBS_YAML_DIR}/omi_aura.yaml ##- !INC ${OBS_YAML_DIR}/ompsnp_npp.yaml ##- !INC ${OBS_YAML_DIR}/ompstc8_npp.yaml ##- !INC ${OBS_YAML_DIR}/cris-fsr_n20.yaml ##- !INC ${OBS_YAML_DIR}/cris-fsr_npp.yaml ##- !INC ${OBS_YAML_DIR}/sfc.yaml -#- !INC ${OBS_YAML_DIR}/sfcship.yaml +##- !INC ${OBS_YAML_DIR}/sfcship.yaml +- !INC ${OBS_YAML_DIR}/gnssro.yaml diff --git a/parm/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.json b/parm/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.json new file mode 100644 index 000000000..17c0421ed --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.json @@ -0,0 +1,34 @@ +{ + "data_format" : "bufr_d", + "data_type" : "gnssro", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : [ "NC003010" ], + "data_description" : "Satellite radio occultation data", + "data_provider" : "U.S. NOAA", + "satellite_info" : [ + { "satellite_name": "MetOp-A", "satellite_full_name": "Meteorological Operational Satellite - A GRAS", "satellite_id": 4 }, + { "satellite_name": "MetOp-B", "satellite_full_name": "Meteorological Operational Satellite - B GRAS", "satellite_id": 3 }, + { "satellite_name": "MetOp-C", "satellite_full_name": "Meteorological Operational Satellite - C GRAS", "satellite_id": 5 }, + { "satellite_name": "TSX", "satellite_full_name": "TerraSAR-X IGOR", "satellite_id": 42 }, + { "satellite_name": "TDX", "satellite_full_name": "TanDEM-X IGOR", "satellite_id": 43 }, + { "satellite_name": "PAZ", "satellite_full_name": "PAZ IGOR", "satellite_id": 44 }, + { "satellite_name": "Sentinel-6A", "satellite_full_name": "Sentinel-6 Michael Freilich", "satellite_id": 66 }, + { "satellite_name": "GeoOptics", "satellite_full_name": "GEOOPTICS CICERO OP1", "satellite_id": 265 }, + { "satellite_name": "GeoOptics", "satellite_full_name": "GEOOPTICS CICERO OP2", "satellite_id": 266 }, + { "satellite_name": "PlanetIQ", "satellite_full_name": "PLANETIQ GNOMES-A", "satellite_id": 267 }, + { "satellite_name": "PlanetIQ", "satellite_full_name": "PLANETIQ GNOMES-B", "satellite_id": 268 }, + { "satellite_name": "Spire", "satellite_full_name": "SPIRE LEMUR 3U CUBESAT", "satellite_id": 269 }, + { "satellite_name": "GraceC", "satellite_full_name": "Gravity Recovery and Climate Experiment C (GRACE-FO)", "satellite_id": 803 }, + { "satellite_name": "GraceD", "satellite_full_name": "Gravity Recovery and Climate Experiment D (GRACE-FO)", "satellite_id": 804 }, + { "satellite_name": "COSMIC-2 E1", "satellite_full_name": "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E1", "satellite_id": 750 }, + { "satellite_name": "COSMIC-2 E2", "satellite_full_name": "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E2", "satellite_id": 751 }, + { "satellite_name": "COSMIC-2 E3", "satellite_full_name": "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E3", "satellite_id": 752 }, + { "satellite_name": "COSMIC-2 E4", "satellite_full_name": "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E4", "satellite_id": 753 }, + { "satellite_name": "COSMIC-2 E5", "satellite_full_name": "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E5", "satellite_id": 754 }, + { "satellite_name": "COSMIC-2 E6", "satellite_full_name": "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E6", "satellite_id": 755 }, + { "satellite_name": "KOMPSAT-5", "satellite_full_name": "Korean Multi-purpose Satellite 5", "satellite_id": 825 } + ] +} diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.py b/ush/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.py new file mode 100755 index 000000000..1b256c4a8 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.py @@ -0,0 +1,650 @@ +#!/usr/bin/env python3 +# (C) Copyright 2023 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import os +import argparse +import json +import numpy as np +import numpy.ma as ma +import math +import calendar +import time +import datetime +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger + +# ==================================================================== +# GPS-RO BUFR dump file +# ===================================================================== +# NC003010 | GPS-RO +# ==================================================================== + + +def Derive_stationIdentification(said, ptid): + + stid = [] + for i in range(len(said)): + newval = str(said[i]).zfill(4)+str(ptid[i]).zfill(4) + stid.append(str(newval)) + stid = np.array(stid).astype(dtype='str') + stid = ma.array(stid) + ma.set_fill_value(stid, "") + + return stid + + +def Compute_Grid_Location(degrees): + + for i in range(len(degrees)): + if degrees[i] <= 360 and degrees[i] >= -180: + degrees[i] = np.deg2rad(degrees[i]) + rad = degrees + + return rad + + +def Compute_imph(impp, elrc): + + imph = (impp - elrc).astype(np.float32) + + return imph + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # ========================================= + # Get parameters from configuration + # ========================================= + data_format = config["data_format"] + data_type = config["data_type"] + bufr_data_type = "gpsro" + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + satellite_info_array = config["satellite_info"] + cycle = config["cycle_datetime"] + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + bufrfile = f"{cycle_type}.t{hh}z.{bufr_data_type}.tm00.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), + 'atmos', bufrfile) + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + logger.debug(f"Making QuerySet ...") + q = bufr.QuerySet(subsets) + + # MetaData + q.add('latitude', '*/ROSEQ1/CLATH') + q.add('longitude', '*/ROSEQ1/CLONH') + q.add('gridLatitude', '*/ROSEQ1/CLATH') + q.add('gridLongitude', '*/ROSEQ1/CLONH') + q.add('year', '*/YEAR') + q.add('year2', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('second', '*/SECO') + q.add('satelliteIdentifier', '*/SAID') + q.add('satelliteInstrument', '*/SIID') + q.add('satelliteConstellationRO', '*/SCLF') + q.add('satelliteTransmitterId', '*/PTID') + q.add('earthRadiusCurvature', '*/ELRC') +# q.add('observationSequenceNum', '*/SEQNUM') + q.add('geoidUndulation', '*/GEODU') + q.add('height', '*/ROSEQ3/HEIT') + q.add('impactParameterRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP') + q.add('impactParameterRO_roseq2repl2', '*/ROSEQ1/ROSEQ2{2}/IMPP') + q.add('impactParameterRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP') + q.add('frequency__roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/MEFR') + q.add('frequency__roseq2repl2', '*/ROSEQ1/ROSEQ2{2}/MEFR') + q.add('frequency__roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/MEFR') +# q.add('pccf', '*/ROSEQ1/PCCF') + q.add('pccf', '*/PCCF[1]') + q.add('percentConfidence', '*/ROSEQ3/PCCF') + q.add('sensorAzimuthAngle', '*/BEARAZ') + + # Processing Center + q.add('dataProviderOrigin', '*/OGCE') + + # Quality Information + q.add('qualityFlags', '*/QFRO') + q.add('qfro', '*/QFRO') + q.add('satelliteAscendingFlag', '*/QFRO') + + # ObsValue + q.add('bendingAngle_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/BNDA[1]') + q.add('bendingAngle_roseq2repl2', '*/ROSEQ1/ROSEQ2{2}/BNDA[1]') + q.add('bendingAngle_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/BNDA[1]') + q.add('atmosphericRefractivity', '*/ROSEQ3/ARFR[1]') + + # ObsError + q.add('obsErrorBendingAngle1', '*/ROSEQ1/ROSEQ2{1}/BNDA[2]') + q.add('obsErrorBendingAngle2', '*/ROSEQ1/ROSEQ2{2}/BNDA[2]') + q.add('obsErrorBendingAngle3', '*/ROSEQ1/ROSEQ2{3}/BNDA[2]') + q.add('obsErrorAtmosphericRefractivity', '*/ROSEQ3/ARFR[2]') + + # ObsType + q.add('obsTypeBendingAngle', '*/SAID') + q.add('obsTypeAtmosphericRefractivity', '*/SAID') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + logger.debug(f" ... Executing QuerySet: get MetaData: basic ...") + # MetaData + clath = r.get('latitude', 'latitude') + clonh = r.get('longitude', 'latitude') + gclath = r.get('gridLatitude', 'latitude') + gclonh = r.get('gridLongitude', 'latitude') + year = r.get('year', 'latitude') + year2 = r.get('year2') + mnth = r.get('month', 'latitude') + days = r.get('day', 'latitude') + hour = r.get('hour', 'latitude') + minu = r.get('minute', 'latitude') + seco = r.get('second', 'latitude') + said = r.get('satelliteIdentifier', 'latitude') + siid = r.get('satelliteInstrument', 'latitude') + sclf = r.get('satelliteConstellationRO', 'latitude') + ptid = r.get('satelliteTransmitterId', 'latitude') + elrc = r.get('earthRadiusCurvature', 'latitude') +# seqnum = r.get('observationSequenceNum') + geodu = r.get('geoidUndulation', 'latitude') + heit = r.get('height', 'height', type='float32').astype(np.float32) + impp1 = r.get('impactParameterRO_roseq2repl1', 'latitude') + impp2 = r.get('impactParameterRO_roseq2repl2', 'latitude') + impp3 = r.get('impactParameterRO_roseq2repl3', 'latitude') + mefr1 = r.get('frequency__roseq2repl1', 'latitude', + type='float32').astype(np.float32) + mefr2 = r.get('frequency__roseq2repl2', 'latitude', + type='float32').astype(np.float32) + mefr3 = r.get('frequency__roseq2repl3', 'latitude', + type='float32').astype(np.float32) + pccf = r.get('pccf', 'latitude', type='float32').astype(np.float32) + ref_pccf = r.get('percentConfidence', 'height') + bearaz = r.get('sensorAzimuthAngle', 'latitude') + + logger.debug(f" ... Executing QuerySet: get MetaData: processing center...") + # Processing Center + ogce = r.get('dataProviderOrigin', 'latitude') + + logger.debug(f" ... Executing QuerySet: get metadata: data quality \ + information ...") + # Quality Information + qfro = r.get('qualityFlags', 'latitude') + qfro2 = r.get('qualityFlags', 'latitude', type='float32').astype(np.float32) + satasc = r.get('satelliteAscendingFlag', 'latitude') + + logger.debug(f" ... Executing QuerySet: get ObsValue: Bending Angle ...") + # ObsValue + # Bending Angle + bnda1 = r.get('bendingAngle_roseq2repl1', 'latitude') + bnda2 = r.get('bendingAngle_roseq2repl2', 'latitude') + bnda3 = r.get('bendingAngle_roseq2repl3', 'latitude') + arfr = r.get('atmosphericRefractivity', 'height') + + # ObsError + # Bending Angle + bndaoe1 = r.get('obsErrorBendingAngle1', 'latitude') + bndaoe2 = r.get('obsErrorBendingAngle2', 'latitude') + bndaoe3 = r.get('obsErrorBendingAngle3', 'latitude') + arfroe = r.get('obsErrorAtmosphericRefractivity', 'height') + + # ObsType + # Bending Angle + bndaot = r.get('obsTypeBendingAngle', 'latitude') + arfrot = r.get('obsTypeBendingAngle', 'latitude') + + logger.debug(f" ... Executing QuerySet: get datatime: observation time ...") + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + timestamp = r.get_datetime('year', 'month', 'day', 'hour', 'minute', + 'second', 'latitude').astype(np.int64) + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + # Check BUFR variable generic dimension and type + logger.debug(f" clath shape, type = {clath.shape}, {clath.dtype}") + logger.debug(f" clonh shape, type = {clonh.shape}, {clonh.dtype}") + logger.debug(f" gclath shape, type = {gclath.shape}, {gclath.dtype}") + logger.debug(f" gclonh shape, type = {gclonh.shape}, {gclonh.dtype}") + logger.debug(f" year shape, type = {year.shape}, {year.dtype}") + logger.debug(f" mnth shape, type = {mnth.shape}, {mnth.dtype}") + logger.debug(f" days shape, type = {days.shape}, {days.dtype}") + logger.debug(f" hour shape, type = {hour.shape}, {hour.dtype}") + logger.debug(f" minu shape, type = {minu.shape}, {minu.dtype}") + logger.debug(f" seco shape, type = {seco.shape}, {seco.dtype}") + logger.debug(f" said shape, type = {said.shape}, {said.dtype}") + logger.debug(f" siid shape, type = {siid.shape}, {siid.dtype}") + logger.debug(f" sclf shape, type = {sclf.shape}, {sclf.dtype}") + logger.debug(f" ptid shape, type = {ptid.shape}, {ptid.dtype}") + logger.debug(f" elrc shape, type = {elrc.shape}, {elrc.dtype}") + logger.debug(f" geodu shape, type = {geodu.shape}, {geodu.dtype}") + logger.debug(f" heit shape, type = {heit.shape}, {heit.dtype}") + logger.debug(f" impp1 shape, type = {impp1.shape}, {impp1.dtype}") + logger.debug(f" impp3 shape, type = {impp3.shape}, {impp3.dtype}") + logger.debug(f" mefr1 shape, type = {mefr1.shape}, {mefr1.dtype}") + logger.debug(f" mefr3 shape, type = {mefr3.shape}, {mefr3.dtype}") + logger.debug(f" pccf shape, type = {pccf.shape}, {pccf.dtype}") + logger.debug(f" pccf shape, fill = {pccf.fill_value}") + logger.debug(f" ref_pccf shape, type = {ref_pccf.shape}, \ + {ref_pccf.dtype}") + logger.debug(f" bearaz shape, type = {bearaz.shape}, {bearaz.dtype}") + + logger.debug(f" ogce shape, type = {ogce.shape}, {ogce.dtype}") + + logger.debug(f" qfro shape, type = {qfro.shape}, {qfro.dtype}") + logger.debug(f" satasc shape, type = {satasc.shape}, {satasc.dtype}") + + logger.debug(f" bnda1 shape, type = {bnda1.shape}, {bnda1.dtype}") + logger.debug(f" bnda3 shape, type = {bnda3.shape}, {bnda3.dtype}") + logger.debug(f" arfr shape, type = {arfr.shape}, {arfr.dtype}") + + logger.debug(f" bndaoe1 shape, type = {bndaoe1.shape}, \ + {bndaoe1.dtype}") + logger.debug(f" bndaoe3 shape, type = {bndaoe3.shape}, \ + {bndaoe3.dtype}") + logger.debug(f" arfroe shape, type = {arfr.shape}, {arfr.dtype}") + + logger.debug(f" bndaot shape, type = {bndaot.shape}, {bndaot.dtype}") + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for executing QuerySet to get ResultSet: \ + {running_time} seconds") + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + logger.debug(f"Creating derived variables - stationIdentification") + stid = Derive_stationIdentification(said, ptid) + + logger.debug(f" stid shape,type = {stid.shape}, {stid.dtype}") + + logger.debug(f"Creating derived variables - Grid Latitude / Longitude ...") + gclonh = Compute_Grid_Location(gclonh) + gclath = Compute_Grid_Location(gclath) + + logger.debug(f" gclonh shape,type = {gclonh.shape}, {gclonh.dtype}") + logger.debug(f" gclath shape,type = {gclath.shape}, {gclath.dtype}") + logger.debug(f" gclonh min/max = {gclonh.min()}, {gclonh.max()}") + logger.debug(f" gclath min/max = {gclath.min()}, {gclath.max()}") + + logger.debug(f"Creating derived variables - imph ...") + imph1 = Compute_imph(impp1, elrc) + imph2 = Compute_imph(impp2, elrc) + imph3 = Compute_imph(impp3, elrc) + + logger.debug(f" imph1 shape,type = {imph1.shape}, {imph1.dtype}") + logger.debug(f" imph3 shape,type = {imph3.shape}, {imph3.dtype}") + logger.debug(f" imph1 min/max = {imph1.min()}, {imph1.max()}") + logger.debug(f" imph3 min/max = {imph3.min()}, {imph3.max()}") + + logger.debug(f"Keep bending angle with Freq = 0.0") + for i in range(len(said)): + if (mefr2[i] == 0.0): + bnda1[i] = bnda2[i] + mefr1[i] = mefr2[i] + impp1[i] = impp2[i] + imph1[i] = imph2[i] + bndaoe1[i] = bndaoe2[i] + if (mefr3[i] == 0.0): + bnda1[i] = bnda3[i] + mefr1[i] = mefr3[i] + impp1[i] = impp3[i] + imph1[i] = imph3[i] + bndaoe1[i] = bndaoe3[i] + + logger.debug(f" new bnda1 shape, type, min/max {bnda1.shape}, \ + {bnda1.dtype}, {bnda1.min()}, {bnda1.max()}") + logger.debug(f" new mefr1 shape, type, min/max {mefr1.shape}, \ + {mefr1.dtype}, {mefr1.min()}, {mefr1.max()}") + logger.debug(f" new impp1 shape, type, min/max {impp1.shape}, \ + {impp1.dtype}, {impp1.min()}, {impp1.max()}") + logger.debug(f" new imph1 shape, type, min/max {imph1.shape}, \ + {imph1.dtype}, {imph1.min()}, {imph1.max()}") + logger.debug(f" new bndaoe1 shape, type, min/max {bndaoe1.shape}, \ + {bndaoe1.dtype}, {bndaoe1.min()}, {bndaoe1.max()}") + +# find ibit for qfro (16bit from left to right) + bit3 = [] + bit5 = [] + bit6 = [] + for quality in qfro: + if quality & 8192 > 0: + bit3.append(1) + else: + bit3.append(0) + + if quality & 2048 > 0: + bit5.append(1) + else: + bit5.append(0) + + if quality & 1024 > 0: + bit6.append(1) + else: + bit6.append(0) + bit3 = np.array(bit3) + bit5 = np.array(bit5) + bit6 = np.array(bit6) + logger.debug(f" new bit3 shape, type, min/max {bit3.shape}, \ + {bit3.dtype}, {bit3.min()}, {bit3.max()}") + +# overwrite satelliteAscendingFlag and QFRO + for quality in range(len(bit3)): + satasc[quality] = 0 + qfro2[quality] = 0.0 + if bit3[quality] == 1: + satasc[quality] = 1 + if (bit5[quality] == 1) or (bit6[quality] == 1): + qfro2[quality] = 1.0 + + logger.debug(f" new satasc shape, type, min/max {satasc.shape}, \ + {satasc.dtype}, {satasc.min()}, {satasc.max()}") + logger.debug(f" new qfro2 shape, type, min/max {qfro2.shape}, \ + {qfro2.dtype}, {qfro2.min()}, {qfro2.max()}, {qfro2.fill_value}") + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for creating derived variables: {running_time} \ + seconds") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Find unique satellite identifiers in data to process + unique_satids = np.unique(said) + logger.debug(f" ... Number of Unique satellite identifiers: \ + {len(unique_satids)}") + logger.debug(f" ... Unique satellite identifiers: {unique_satids}") + + # Create the dimensions + dims = {'Location': np.arange(0, clath.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}.tm00.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + # Create IODA ObsSpace + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + obsspace.write_attr('data_format', data_format) + obsspace.write_attr('data_type', data_type) + obsspace.write_attr('subsets', subsets) + obsspace.write_attr('cycle_type', cycle_type) + obsspace.write_attr('cycle_datetime', cycle) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('converter', os.path.basename(__file__)) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, & attributes") + # Longitude + obsspace.create_var('MetaData/longitude', dtype=clonh.dtype, + fillval=clonh.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(clonh) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=clath.dtype, + fillval=clath.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(clath) + + # Grid Longitude + obsspace.create_var('MetaData/gridLongitude', dtype=gclonh.dtype, + fillval=gclonh.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('valid_range', np.array([-3.14159265, 3.14159265], + dtype=np.float32)) \ + .write_attr('long_name', 'Grid Longitude') \ + .write_data(gclonh) + + # Grid Latitude + obsspace.create_var('MetaData/gridLatitude', dtype=gclath.dtype, + fillval=gclath.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('valid_range', np.array([-1.570796325, 1.570796325], + dtype=np.float32)) \ + .write_attr('long_name', 'Grid Latitude') \ + .write_data(gclath) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=np.int64, + fillval=timestamp.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(timestamp) + + # Station Identification + obsspace.create_var('MetaData/stationIdentification', dtype=stid.dtype, + fillval=stid.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(stid) + + # Satellite Identifier + obsspace.create_var('MetaData/satelliteIdentifier', dtype=said.dtype, + fillval=said.fill_value) \ + .write_attr('long_name', 'Satellite Identifier') \ + .write_data(said) + + # Satellite Instrument + obsspace.create_var('MetaData/satelliteInstrument', dtype=siid.dtype, + fillval=siid.fill_value) \ + .write_attr('long_name', 'Satellite Instrument') \ + .write_data(siid) + + # Satellite Constellation RO + obsspace.create_var('MetaData/satelliteConstellationRO', dtype=sclf.dtype, + fillval=sclf.fill_value) \ + .write_attr('long_name', 'Satellite Constellation RO') \ + .write_data(sclf) + + # Satellite Transmitter ID + obsspace.create_var('MetaData/satelliteTransmitterId', dtype=ptid.dtype, + fillval=ptid.fill_value) \ + .write_attr('long_name', 'Satellite Transmitter Id') \ + .write_data(ptid) + + # Earth Radius Curvature + obsspace.create_var('MetaData/earthRadiusCurvature', dtype=elrc.dtype, + fillval=elrc.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Earth Radius of Curvature') \ + .write_data(elrc) + + # Geoid Undulation + obsspace.create_var('MetaData/geoidUndulation', dtype=geodu.dtype, + fillval=geodu.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Geoid Undulation') \ + .write_data(geodu) + + # Height + obsspace.create_var('MetaData/height', dtype=heit.dtype, + fillval=heit.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Height') \ + .write_data(heit) + + # Impact Parameter RO + obsspace.create_var('MetaData/impactParameterRO', dtype=impp1.dtype, + fillval=impp1.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Impact Parameter RO') \ + .write_data(impp1) + + # Impact Height RO + obsspace.create_var('MetaData/impactHeightRO', dtype=imph1.dtype, + fillval=imph1.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Impact Height RO') \ + .write_data(imph1) + + # Impact Height RO + obsspace.create_var('MetaData/frequency', dtype=mefr1.dtype, + fillval=mefr1.fill_value) \ + .write_attr('units', 'Hz') \ + .write_attr('long_name', 'Frequency') \ + .write_data(mefr1) + + # PCCF Percent Confidence + obsspace.create_var('MetaData/pccf', dtype=pccf.dtype, + fillval=pccf.fill_value) \ + .write_attr('units', '%') \ + .write_attr('long_name', 'Profile Percent Confidence') \ + .write_data(pccf) + + # PCCF Ref Percent Confidence + obsspace.create_var('MetaData/percentConfidence', dtype=ref_pccf.dtype, + fillval=ref_pccf.fill_value) \ + .write_attr('units', '%') \ + .write_attr('long_name', 'Ref Percent Confidence') \ + .write_data(ref_pccf) + + # Azimuth Angle + obsspace.create_var('MetaData/sensorAzimuthAngle', dtype=bearaz.dtype, + fillval=bearaz.fill_value) \ + .write_attr('units', 'degree') \ + .write_attr('long_name', 'Percent Confidence') \ + .write_data(bearaz) + + # Data Provider + obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce.dtype, + fillval=ogce.fill_value) \ + .write_attr('long_name', 'Identification of Originating Center') \ + .write_data(ogce) + + # Quality: Quality Flags + obsspace.create_var('MetaData/qfro', dtype=qfro.dtype, + fillval=qfro.fill_value) \ + .write_attr('long_name', 'QFRO') \ + .write_data(qfro) + + obsspace.create_var('MetaData/qualityFlags', dtype=qfro2.dtype, + fillval=qfro2.fill_value) \ + .write_attr('long_name', 'Quality Flags for QFRO bit5 and bit6') \ + .write_data(qfro2) + + # Quality: Satellite Ascending Flag + obsspace.create_var('MetaData/satelliteAscendingFlag', dtype=satasc.dtype, + fillval=satasc.fill_value) \ + .write_attr('long_name', 'Satellite Ascending Flag') \ + .write_data(satasc) + + # ObsValue: Bending Angle + obsspace.create_var('ObsValue/bendingAngle', dtype=bnda1.dtype, + fillval=bnda1.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('long_name', 'Bending Angle') \ + .write_data(bnda1) + + # ObsValue: Atmospheric Refractivity + obsspace.create_var('ObsValue/atmosphericRefractivity', dtype=arfr.dtype, + fillval=arfr.fill_value) \ + .write_attr('units', 'N-units') \ + .write_attr('long_name', 'Atmospheric Refractivity') \ + .write_data(arfr) + + # ObsError: Bending Angle + obsspace.create_var('ObsError/bendingAngle', dtype=bndaoe1.dtype, + fillval=bndaoe1.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('long_name', 'Bending Angle Obs Error') \ + .write_data(bndaoe1) + + # ObsError: Atmospheric Refractivity + obsspace.create_var('ObsError/atmosphericRefractivity', dtype=arfroe.dtype, + fillval=arfroe.fill_value) \ + .write_attr('units', 'N-units') \ + .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ + .write_data(arfroe) + + # ObsType: Bending Angle + obsspace.create_var('ObsType/BendingAngle', dtype=bndaot.dtype, + fillval=bndaot.fill_value) \ + .write_attr('long_name', 'Bending Angle ObsType') \ + .write_data(bndaot) + + # ObsType: Atmospheric Refractivity + obsspace.create_var('ObsType/atmosphericRefractivity', dtype=arfrot.dtype, + fillval=arfrot.fill_value) \ + .write_attr('long_name', 'Atmospheric Refractivity ObsType') \ + .write_data(arfrot) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA for gnssro bufr: \ + {running_time} seconds") + + logger.debug("All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, + help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', + help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_gnssro.py', level=log_level, + colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") From 047e91415db11dda3534dd57ff2ee214fc7018d8 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Mon, 20 Nov 2023 16:53:01 -0500 Subject: [PATCH 26/58] Update for ATMS for End-to-End Test (#757) List of items to be added in this PR: - BUFR Converter: Add YAML for converting ATMS BUFR to IODA - The ObsValue is antenna temperature - Input BUFR is normal feel - Testing YAML: re-evaluation results due to update CRTM from v2.3.0 to v2.4.1-jedi.1 - atms_npp_noqc.yaml --- 100% replication - atms_npp.yaml --- 100 % replication - atms_n20_noqc.yaml --- 100 % replication - atms_n20.yaml --- ~ 1100% replication: two channel 7 observations difference - Config YAML: * add two read routine QC: data thinning and removal of data from scan edge - atms_npp.yaml - atms_n20.yaml - Paring PR in JCSDA UFO repository - This PR also has a [paring PR in JCSDA UFO repository](https://github.com/JCSDA-internal/ufo/pull/3094). - It is OK to merge this one into GDASApp before the paring UFO PR - The related test data (obs and geoval files from GSI) for UFO Evaluation have been updated (due to CRTM-2.4.1 update) on ORION and HERA. Notes: - End-to-end testing completed without problem. - However, the filter results show one of the filters, 88-165 GHz scattering index check, tossed more data compared to GSI. - The excess screening from the 88-165 GHz scattering index check is under investigation and will be reported separately. - Please check comments below for validation results --- parm/atm/obs/config/atms_n20.yaml | 595 ++++++++++++++-------- parm/atm/obs/config/atms_npp.yaml | 530 +++++++++++++++++++ parm/atm/obs/lists/gdas_prototype_3d.yaml | 8 +- parm/atm/obs/testing/atms_n20.yaml | 146 +++--- parm/atm/obs/testing/atms_n20_noqc.yaml | 145 ++++++ parm/atm/obs/testing/atms_npp.yaml | 149 +++--- parm/atm/obs/testing/atms_npp_noqc.yaml | 145 ++++++ parm/ioda/bufr2ioda/bufr2ioda_atms.yaml | 6 +- 8 files changed, 1360 insertions(+), 364 deletions(-) create mode 100644 parm/atm/obs/config/atms_npp.yaml create mode 100644 parm/atm/obs/testing/atms_n20_noqc.yaml create mode 100644 parm/atm/obs/testing/atms_npp_noqc.yaml diff --git a/parm/atm/obs/config/atms_n20.yaml b/parm/atm/obs/config/atms_n20.yaml index 28b19e692..93590ab7a 100644 --- a/parm/atm/obs/config/atms_n20.yaml +++ b/parm/atm/obs/config/atms_n20.yaml @@ -3,24 +3,26 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)atms_n20.${{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)atms_n20.tm00.nc obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_atms_n20_${{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_atms_n20_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 - simulated variables: [brightness_temperature] + simulated variables: [brightnessTemperature] channels: &atms_n20_channels 1-22 + obs operator: name: CRTM Absorbers: [H2O,O3] Clouds: [Water, Ice] Cloud_Fraction: 1.0 obs options: - Sensor_ID: atms_n20 + Sensor_ID: &Sensor_ID atms_n20 EndianType: little_endian CoefficientPath: $(DATA)/crtm/ + obs bias: input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias.nc4 output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias.nc4 @@ -51,24 +53,139 @@ obs bias: ratio: 1.1 ratio for small dataset: 2.0 output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias_cov.nc4 -obs filters: -- filter: BlackList + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels - action: - name: assign error - error function: + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: SIRetFromObs@DerivedMetaData + type: float + function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 16 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: CLWMatchIndex@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: CLWMatchIndexMW@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwret_clearsky: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + type: float + function: name: ObsErrorModelRamp@ObsFunction channels: *atms_n20_channels options: channels: *atms_n20_channels xvar: - name: CLWRetSymmetricMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] + name: CLWRetSymmetric@DerivedMetaData x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, @@ -81,293 +198,333 @@ obs filters: 0.500, 0.500] err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, + 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, 3.000, 3.000] err1: [20.000, 25.000, 12.000, 7.000, 3.500, 3.000, 0.800, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, + 0.450, 0.450, 0.550, 0.800, 4.000, 19.000, 30.000, 25.000, 16.500, 12.000, 9.000, 6.500] -# CLW Retrieval Check + +# Calculate Innovation@DerivedMetaData +- filter: Variable Assignment + assignments: + - name: Innovation@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *atms_n20_channels + options: + variables: + - name: brightnessTemperature@ObsValue + channels: *atms_n20_channels + - name: brightnessTemperature@HofX + channels: *atms_n20_channels + coefs: [1, -1] + +# Step 0-C: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + +# Step 1: Remove observations from the edge of the scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-22 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 7-90 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +#- filter: Gaussian Thinning +# horizontal_mesh: 145 +# use_reduced_horizontal_grid: true +## round_horizontal_bin_count_to_nearest: true +## partition_longitude_bins_using_mesh: true +# actions: +# - name: set +# flag: Thinning +# - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check filter variables: - - name: brightness_temperature - channels: 1-7, 16-22 + - name: brightnessTemperature + channels: 1-7, 16-22 test variables: - - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] + - name: CLWRetFromObs@DerivedMetaData maxvalue: 999.0 - action: - name: reject -# CLW Retrieval Check + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 3B: CLW Retrieval Check (background_based) - filter: Bounds Check filter variables: - - name: brightness_temperature - channels: 1-7, 16-22 + - name: brightnessTemperature + channels: 1-7, 16-22 test variables: - - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] + - name: CLWRetFromBkg@DerivedMetaData maxvalue: 999.0 - action: - name: reject -# Hydrometeor Check (cloud/precipitation affected chanels) + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 4: Window channel sanity check - filter: Bounds Check filter variables: - - name: brightness_temperature - channels: *atms_n20_channels + - name: brightnessTemperature + channels: 1-7, 16, 17-22 test variables: - - name: HydrometeorCheckATMS@ObsFunction + - name: Innovation@DerivedMetaData + channels: 1, 2, 5-7, 16 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/HydrometeorCheckATMS channels: *atms_n20_channels - options: + type: float + function: + name: HydrometeorCheckATMS@ObsFunction channels: *atms_n20_channels - obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - clwret_function: - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - obserr_function: - name: ObsErrorModelRamp@ObsFunction + options: channels: *atms_n20_channels - options: + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData channels: *atms_n20_channels - xvar: - name: CLWRetSymmetricMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, - 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] - x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, - 1.000, 1.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.350, 0.500, 0.500, 0.500, 0.500, - 0.500, 0.500] - err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - err1: [20.000, 25.000, 12.000, 7.000, 3.500, - 3.000, 0.800, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 19.000, 30.000, 25.000, 16.500, 12.000, - 9.000, 6.500] - maxvalue: 0.0 - action: - name: reject -# Topography check -- filter: BlackList + +- filter: Bounds Check filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels - action: - name: inflate error - inflation variable: + test variables: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_n20_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_n20_channels + type: float + function: name: ObsErrorFactorTopoRad@ObsFunction channels: *atms_n20_channels options: - sensor: atms_n20 + sensor: *Sensor_ID channels: *atms_n20_channels -# Transmittnace Top Check -- filter: BlackList + +- filter: Perform Action filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels action: name: inflate error inflation variable: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_n20_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_n20_channels + type: float + function: name: ObsErrorFactorTransmitTopRad@ObsFunction channels: *atms_n20_channels options: channels: *atms_n20_channels -# Surface Jacobian check -- filter: BlackList + +- filter: Perform Action filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels action: name: inflate error inflation variable: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_n20_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *atms_n20_channels + type: float + function: name: ObsErrorFactorSurfJacobianRad@ObsFunction channels: *atms_n20_channels options: + sensor: *Sensor_ID channels: *atms_n20_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] -# Situation dependent Check -- filter: BlackList + + +- filter: Perform Action filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels action: name: inflate error inflation variable: + name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *atms_n20_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *atms_n20_channels + type: float + function: name: ObsErrorFactorSituDependMW@ObsFunction channels: *atms_n20_channels options: - sensor: atms_n20 + sensor: *Sensor_ID channels: *atms_n20_channels - clwobs_function: - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] clwbkg_function: - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] + name: CLWRetFromBkg@DerivedMetaData + clwobs_function: + name: CLWRetFromObs@DerivedMetaData scatobs_function: - name: SCATRetMW@ObsFunction - options: - scatret_ch238: 1 - scatret_ch314: 2 - scatret_ch890: 16 - scatret_types: [ObsValue] + name: SIRetFromObs@DerivedMetaData clwmatchidx_function: - name: CLWMatchIndexMW@ObsFunction + name: CLWMatchIndex@DerivedMetaData + channels: *atms_n20_channels + obserr_function: + name: InitialObsError@DerivedMetaData channels: *atms_n20_channels - options: - channels: *atms_n20_channels - clwobs_function: - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - clwbkg_function: - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - clwret_clearsky: [ 0.030, 0.030, 0.030, 0.020, 0.030, - 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, + 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, 3.000, 3.000] -# Gross check -- filter: Background Check + +- filter: Perform Action filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels - function absolute threshold: - - name: ObsErrorBoundMW@ObsFunction + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *atms_n20_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorLat@DerivedMetaData + type: float + function: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: ObsErrorBound@DerivedMetaData channels: *atms_n20_channels - options: - sensor: atms_n20 + type: float + function: + name: ObsErrorBoundMW@ObsFunction channels: *atms_n20_channels - obserr_bound_latitude: - name: ObsErrorFactorLatRad@ObsFunction - options: - latitude_parameters: [25.0, 0.25, 0.04, 3.0] - obserr_bound_transmittop: - name: ObsErrorFactorTransmitTopRad@ObsFunction + options: + sensor: *Sensor_ID channels: *atms_n20_channels - options: + obserr_bound_latitude: + name: ObsErrorFactorLat@DerivedMetaData + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTop@DerivedMetaData channels: *atms_n20_channels - obserr_bound_topo: - name: ObsErrorFactorTopoRad@ObsFunction - channels: *atms_n20_channels - options: + options: + channels: *atms_n20_channels + obserr_bound_topo: + name: ObsErrorFactorTopo@DerivedMetaData channels: *atms_n20_channels - sensor: atms_n20 - obserr_function: - name: ObsErrorModelRamp@ObsFunction - channels: *atms_n20_channels - options: + obserr_function: + name: InitialObsError@DerivedMetaData channels: *atms_n20_channels - xvar: - name: CLWRetSymmetricMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, - 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] - x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, - 1.000, 1.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.350, 0.500, 0.500, 0.500, 0.500, - 0.500, 0.500] - err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - err1: [20.000, 25.000, 12.000, 7.000, 3.500, - 3.000, 0.800, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 19.000, 30.000, 25.000, 16.500, 12.000, - 9.000, 6.500] - obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 2.0, 4.5, - 4.5, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0] - action: - name: reject -# Inter-channel check -- filter: Bounds Check + threhold: 3 + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 2.0, 4.5, + 4.5, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0] + +- filter: Background Check filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels - test variables: - - name: InterChannelConsistencyCheck@ObsFunction + function absolute threshold: + - name: ObsErrorBound@DerivedMetaData channels: *atms_n20_channels - options: - channels: *atms_n20_channels - sensor: atms_n20 - use_flag: [ 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1, - 1, 1, 1, 1, 1, - 1, 1] - maxvalue: 1.0e-12 - action: - name: reject -# Useflag check + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check - filter: Bounds Check filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels test variables: - - name: ChannelUseflagCheckRad@ObsFunction + - name: InterChannelConsistencyCheck@ObsFunction channels: *atms_n20_channels options: channels: *atms_n20_channels + use passive_bc: true + sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] - minvalue: 1.0e-12 - action: - name: reject + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + diff --git a/parm/atm/obs/config/atms_npp.yaml b/parm/atm/obs/config/atms_npp.yaml new file mode 100644 index 000000000..bb668ab9d --- /dev/null +++ b/parm/atm/obs/config/atms_npp.yaml @@ -0,0 +1,530 @@ +obs space: + name: atms_npp + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)atms_npp.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_atms_npp_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &atms_npp_channels 1-22 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_npp + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)atms_npp.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)atms_npp.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &atms_npp_tlapse $(DATA)/obs/$(GPREFIX)atms_npp.tlapse.txt + - name: lapse_rate + tlapse: *atms_npp_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)atms_npp.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)atms_npp.satbias_cov.nc4 + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: SIRetFromObs@DerivedMetaData + type: float + function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 16 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: CLWMatchIndex@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: CLWMatchIndexMW@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwret_clearsky: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsErrorModelRamp@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + xvar: + name: CLWRetSymmetric@DerivedMetaData + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] + +# Calculate Innovation@DerivedMetaData +- filter: Variable Assignment + assignments: + - name: Innovation@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *atms_npp_channels + options: + variables: + - name: brightnessTemperature@ObsValue + channels: *atms_npp_channels + - name: brightnessTemperature@HofX + channels: *atms_npp_channels + coefs: [1, -1] + +# Step 0-C: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + +# Step 1: Remove observations from the edge of the scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-22 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 7-90 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +#- filter: Gaussian Thinning +# horizontal_mesh: 145 +# use_reduced_horizontal_grid: true +## round_horizontal_bin_count_to_nearest: true +## partition_longitude_bins_using_mesh: true +# actions: +# - name: set +# flag: Thinning +# - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromObs@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 3B: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromBkg@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16, 17-22 + test variables: + - name: Innovation@DerivedMetaData + channels: 1, 2, 5-7, 16 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_npp_channels + type: float + function: + name: HydrometeorCheckATMS@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + test variables: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_npp_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *atms_npp_channels + options: + sensor: *Sensor_ID + channels: *atms_npp_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_npp_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_npp_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *atms_npp_channels + options: + sensor: *Sensor_ID + channels: *atms_npp_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *atms_npp_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *atms_npp_channels + options: + sensor: *Sensor_ID + channels: *atms_npp_channels + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + scatobs_function: + name: SIRetFromObs@DerivedMetaData + clwmatchidx_function: + name: CLWMatchIndex@DerivedMetaData + channels: *atms_npp_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *atms_npp_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorLat@DerivedMetaData + type: float + function: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: ObsErrorBound@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsErrorBoundMW@ObsFunction + channels: *atms_npp_channels + options: + sensor: *Sensor_ID + channels: *atms_npp_channels + obserr_bound_latitude: + name: ObsErrorFactorLat@DerivedMetaData + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + obserr_bound_topo: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_npp_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 2.0, 4.5, + 4.5, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + function absolute threshold: + - name: ObsErrorBound@DerivedMetaData + channels: *atms_npp_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml b/parm/atm/obs/lists/gdas_prototype_3d.yaml index 98877f47c..e8f2d006c 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml @@ -2,8 +2,12 @@ observers: ##- !INC ${OBS_YAML_DIR}/amsua_n19.yaml ##- !INC ${OBS_YAML_DIR}/sondes_prepbufr.yaml ##- !INC ${OBS_YAML_DIR}/atms_n20.yaml +##- !INC ${OBS_YAML_DIR}/atms_npp.yaml ##- !INC ${OBS_YAML_DIR}/aircraft.yaml -##- !INC ${OBS_YAML_DIR}/satwind_goes-16.yaml +- !INC ${OBS_YAML_DIR}/satwind_goes-16.yaml +##- !INC ${OBS_YAML_DIR}/satwind_goes-17.yaml +##- !INC ${OBS_YAML_DIR}/scatwind_metop-a.yaml +##- !INC ${OBS_YAML_DIR}/scatwind_metop-b.yaml ##- !INC ${OBS_YAML_DIR}/omi_aura.yaml ##- !INC ${OBS_YAML_DIR}/ompsnp_npp.yaml ##- !INC ${OBS_YAML_DIR}/ompstc8_npp.yaml @@ -11,4 +15,4 @@ observers: ##- !INC ${OBS_YAML_DIR}/cris-fsr_npp.yaml ##- !INC ${OBS_YAML_DIR}/sfc.yaml ##- !INC ${OBS_YAML_DIR}/sfcship.yaml -- !INC ${OBS_YAML_DIR}/gnssro.yaml +##- !INC ${OBS_YAML_DIR}/gnssro.yaml \ No newline at end of file diff --git a/parm/atm/obs/testing/atms_n20.yaml b/parm/atm/obs/testing/atms_n20.yaml index 1eaf198ce..9f1c913b0 100644 --- a/parm/atm/obs/testing/atms_n20.yaml +++ b/parm/atm/obs/testing/atms_n20.yaml @@ -18,7 +18,7 @@ obs space: type: H5File obsfile: !ENV atms_n20_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-22 + channels: &atms_n20_channels 1-22 geovals: filename: !ENV atms_n20_geoval_${CDATE}.nc4 obs bias: @@ -46,7 +46,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels flags: - name: CLWRetrievalReject initial value: false @@ -56,7 +56,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels flags: - name: HydrometeorCheckReject initial value: false @@ -66,7 +66,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels flags: - name: GrossCheckReject initial value: false @@ -76,7 +76,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels flags: - name: InterChannelCheckReject initial value: false @@ -153,13 +153,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: CLWMatchIndex@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: CLWMatchIndexMW@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: - channels: *all_channels + channels: *atms_n20_channels clwobs_function: name: CLWRetFromObs@DerivedMetaData clwbkg_function: @@ -174,13 +174,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: ObsErrorModelRamp@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: - channels: *all_channels + channels: *atms_n20_channels xvar: name: CLWRetSymmetric@DerivedMetaData x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, @@ -208,29 +208,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: Innovation@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *atms_n20_channels options: variables: - name: brightnessTemperature@ObsValue - channels: *all_channels + channels: *atms_n20_channels - name: brightnessTemperature@HofX - channels: *all_channels + channels: *atms_n20_channels coefs: [1, -1] # Step 1: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels action: name: assign error error function: name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -271,25 +271,34 @@ obs post filters: flag all filter variables if any test variable is out of bounds: true # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_n20_channels + type: float + function: + name: HydrometeorCheckATMS@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels test variables: - - name: HydrometeorCheckATMS@ObsFunction - channels: *all_channels - options: - channels: *all_channels - obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - clwret_function: - name: CLWRetFromObs@DerivedMetaData - obserr_function: - name: InitialObsError@DerivedMetaData - channels: *all_channels + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_n20_channels maxvalue: 0.0 actions: - name: set @@ -301,59 +310,59 @@ obs post filters: - filter: Variable Assignment assignments: - name: ObsErrorFactorTopo@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: ObsErrorFactorTopoRad@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_n20_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels action: name: inflate error inflation variable: name: ObsErrorFactorTopo@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels # Step 7: Obs error inflation based on TOA transmittancec check - filter: Variable Assignment assignments: - name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: ObsErrorFactorTransmitTopRad@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: - channels: *all_channels + channels: *atms_n20_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels action: name: inflate error inflation variable: name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels # Step 8: Observation error inflation based on surface jacobian check - filter: Variable Assignment assignments: - name: ObsErrorFactorSurfJacobian@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: ObsErrorFactorSurfJacobianRad@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_n20_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] @@ -361,25 +370,25 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels action: name: inflate error inflation variable: name: ObsErrorFactorSurfJacobian@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels # Step 9: Situation dependent check - filter: Variable Assignment assignments: - name: ObsErrorFactorSituDepend@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: ObsErrorFactorSituDependMW@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_n20_channels clwbkg_function: name: CLWRetFromBkg@DerivedMetaData clwobs_function: @@ -388,25 +397,25 @@ obs post filters: name: SIRetFromObs@DerivedMetaData clwmatchidx_function: name: CLWMatchIndex@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels obserr_function: name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, + 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, 3.000, 3.000] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels action: name: inflate error inflation variable: name: ObsErrorFactorSituDepend@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels # Step 10: Gross check # Remove data if abs(Obs-HofX) > absolute threhold @@ -422,27 +431,27 @@ obs post filters: - filter: Variable Assignment assignments: - name: ObsErrorBound@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: ObsErrorBoundMW@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_n20_channels obserr_bound_latitude: name: ObsErrorFactorLat@DerivedMetaData obserr_bound_transmittop: name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels options: - channels: *all_channels + channels: *atms_n20_channels obserr_bound_topo: name: ObsErrorFactorTopo@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels obserr_function: name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, @@ -453,10 +462,10 @@ obs post filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels function absolute threshold: - name: ObsErrorBound@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels actions: - name: set flag: GrossCheckReject @@ -467,12 +476,12 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels test variables: - name: InterChannelConsistencyCheck@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: - channels: *all_channels + channels: *atms_n20_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, @@ -486,6 +495,5 @@ obs post filters: flag: InterChannelCheckReject ignore: rejected observations - name: reject -passedBenchmark: 171071 # GSI: 171076 -# Notes: one point difference in HydrometeorCheck for channel 7 +passedBenchmark: 181401 # GSI: 181403 (difference is in channel 7) diff --git a/parm/atm/obs/testing/atms_n20_noqc.yaml b/parm/atm/obs/testing/atms_n20_noqc.yaml new file mode 100644 index 000000000..1d9a9dae1 --- /dev/null +++ b/parm/atm/obs/testing/atms_n20_noqc.yaml @@ -0,0 +1,145 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_n20 + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: atms_n20 + obsdatain: + engine: + type: H5File + obsfile: !ENV atms_n20_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV atms_n20_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &atms_n20_channels 1-22 +geovals: + filename: !ENV atms_n20_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV atms_n20_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &atms_n20_tlap !ENV atms_n20_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *atms_n20_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsErrorModelRamp@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + xvar: + name: CLWRetSymmetric@DerivedMetaData + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + +passedBenchmark: 249449 +#vector ref: GsiHofXBc +#tolerance: 1.e-7 diff --git a/parm/atm/obs/testing/atms_npp.yaml b/parm/atm/obs/testing/atms_npp.yaml index 6558d79cc..beae69095 100644 --- a/parm/atm/obs/testing/atms_npp.yaml +++ b/parm/atm/obs/testing/atms_npp.yaml @@ -18,7 +18,7 @@ obs space: type: H5File obsfile: !ENV atms_npp_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-22 + channels: &atms_npp_channels 1-22 geovals: filename: !ENV atms_npp_geoval_${CDATE}.nc4 obs bias: @@ -46,7 +46,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels flags: - name: CLWRetrievalReject initial value: false @@ -56,7 +56,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels flags: - name: HydrometeorCheckReject initial value: false @@ -66,7 +66,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels flags: - name: GrossCheckReject initial value: false @@ -76,7 +76,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels flags: - name: InterChannelCheckReject initial value: false @@ -153,13 +153,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: CLWMatchIndex@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: CLWMatchIndexMW@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: - channels: *all_channels + channels: *atms_npp_channels clwobs_function: name: CLWRetFromObs@DerivedMetaData clwbkg_function: @@ -174,13 +174,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: ObsErrorModelRamp@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: - channels: *all_channels + channels: *atms_npp_channels xvar: name: CLWRetSymmetric@DerivedMetaData x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, @@ -208,29 +208,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: Innovation@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *atms_npp_channels options: variables: - name: brightnessTemperature@ObsValue - channels: *all_channels + channels: *atms_npp_channels - name: brightnessTemperature@HofX - channels: *all_channels + channels: *atms_npp_channels coefs: [1, -1] # Step 1: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels action: name: assign error error function: name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -271,25 +271,34 @@ obs post filters: flag all filter variables if any test variable is out of bounds: true # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_npp_channels + type: float + function: + name: HydrometeorCheckATMS@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels test variables: - - name: HydrometeorCheckATMS@ObsFunction - channels: *all_channels - options: - channels: *all_channels - obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - clwret_function: - name: CLWRetFromObs@DerivedMetaData - obserr_function: - name: InitialObsError@DerivedMetaData - channels: *all_channels + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_npp_channels maxvalue: 0.0 actions: - name: set @@ -301,59 +310,59 @@ obs post filters: - filter: Variable Assignment assignments: - name: ObsErrorFactorTopo@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: ObsErrorFactorTopoRad@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_npp_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels action: name: inflate error inflation variable: name: ObsErrorFactorTopo@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels # Step 7: Obs error inflation based on TOA transmittancec check - filter: Variable Assignment assignments: - name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: ObsErrorFactorTransmitTopRad@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: - channels: *all_channels + channels: *atms_npp_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels action: name: inflate error inflation variable: name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels # Step 8: Observation error inflation based on surface jacobian check - filter: Variable Assignment assignments: - name: ObsErrorFactorSurfJacobian@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: ObsErrorFactorSurfJacobianRad@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_npp_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] @@ -361,25 +370,25 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels action: name: inflate error inflation variable: name: ObsErrorFactorSurfJacobian@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels # Step 9: Situation dependent check - filter: Variable Assignment assignments: - name: ObsErrorFactorSituDepend@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: ObsErrorFactorSituDependMW@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_npp_channels clwbkg_function: name: CLWRetFromBkg@DerivedMetaData clwobs_function: @@ -388,25 +397,25 @@ obs post filters: name: SIRetFromObs@DerivedMetaData clwmatchidx_function: name: CLWMatchIndex@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels obserr_function: name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, + 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, 3.000, 3.000] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels action: name: inflate error inflation variable: name: ObsErrorFactorSituDepend@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels # Step 10: Gross check # Remove data if abs(Obs-HofX) > absolute threhold @@ -422,41 +431,40 @@ obs post filters: - filter: Variable Assignment assignments: - name: ObsErrorBound@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: ObsErrorBoundMW@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_npp_channels obserr_bound_latitude: name: ObsErrorFactorLat@DerivedMetaData obserr_bound_transmittop: name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels options: - channels: *all_channels + channels: *atms_npp_channels obserr_bound_topo: name: ObsErrorFactorTopo@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels obserr_function: name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] - - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels function absolute threshold: - name: ObsErrorBound@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels actions: - name: set flag: GrossCheckReject @@ -467,16 +475,16 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels test variables: - name: InterChannelConsistencyCheck@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: - channels: *all_channels + channels: *atms_npp_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] @@ -486,6 +494,5 @@ obs post filters: flag: InterChannelCheckReject ignore: rejected observations - name: reject -passedBenchmark: 169997 # GSI: 169998 -# Notes: one point difference in HydrometeorCheck for channel 7 +passedBenchmark: 182751 # GSI: 182751 diff --git a/parm/atm/obs/testing/atms_npp_noqc.yaml b/parm/atm/obs/testing/atms_npp_noqc.yaml new file mode 100644 index 000000000..f8afbb175 --- /dev/null +++ b/parm/atm/obs/testing/atms_npp_noqc.yaml @@ -0,0 +1,145 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_npp + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: atms_npp + obsdatain: + engine: + type: H5File + obsfile: !ENV atms_npp_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV atms_npp_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &atms_npp_channels 1-22 +geovals: + filename: !ENV atms_npp_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV atms_npp_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &atms_npp_tlap !ENV atms_npp_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *atms_npp_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsErrorModelRamp@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + xvar: + name: CLWRetSymmetric@DerivedMetaData + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + +passedBenchmark: 247947 +#vector ref: GsiHofXBc +##tolerance: 1.e-7 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml b/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml index 39fa399ab..0f1d69fb7 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml +++ b/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml @@ -31,7 +31,7 @@ observations: # However, UFO/CRTM requred this variable to be float fieldOfViewNumber: query: "*/FOVN" - type: float +# type: float heightOfStation: query: "*/HMSL" @@ -64,7 +64,7 @@ observations: remappedBrightnessTemperature: fieldOfViewNumber: "*/FOVN" sensorChannelNumber: "*/ATMSCH/CHNM" - brightnessTemperature: "*/ATMSCH/TMBR" + brightnessTemperature: "*/ATMSCH/TMANT" obsTime: year: "*/YEAR" month: "*/MNTH" @@ -84,7 +84,7 @@ observations: ioda: backend: netcdf - obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.atms.$(splitvar).tm00.nc" + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.atms_$(splitvar).tm00.nc" dimensions: - name: Channel From abe9fcef7e477a486a81d048740c91ab438ff432 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Mon, 20 Nov 2023 23:24:18 -0600 Subject: [PATCH 27/58] Update amsua obs YAML files --- parm/atm/obs/testing/amsua_metop-a.yaml | 112 +++++++-------- parm/atm/obs/testing/amsua_metop-a_noqc.yaml | 138 +++++++++++++++++++ parm/atm/obs/testing/amsua_metop-b.yaml | 112 +++++++-------- parm/atm/obs/testing/amsua_metop-b_noqc.yaml | 137 ++++++++++++++++++ parm/atm/obs/testing/amsua_metop-c.yaml | 112 +++++++-------- parm/atm/obs/testing/amsua_metop-c_noqc.yaml | 138 +++++++++++++++++++ parm/atm/obs/testing/amsua_n15.yaml | 112 +++++++-------- parm/atm/obs/testing/amsua_n15_noqc.yaml | 137 ++++++++++++++++++ parm/atm/obs/testing/amsua_n18.yaml | 112 +++++++-------- parm/atm/obs/testing/amsua_n18_noqc.yaml | 137 ++++++++++++++++++ parm/atm/obs/testing/amsua_n19.yaml | 112 +++++++-------- parm/atm/obs/testing/amsua_n19_noqc.yaml | 137 ++++++++++++++++++ 12 files changed, 1160 insertions(+), 336 deletions(-) create mode 100644 parm/atm/obs/testing/amsua_metop-a_noqc.yaml create mode 100644 parm/atm/obs/testing/amsua_metop-b_noqc.yaml create mode 100644 parm/atm/obs/testing/amsua_metop-c_noqc.yaml create mode 100644 parm/atm/obs/testing/amsua_n15_noqc.yaml create mode 100644 parm/atm/obs/testing/amsua_n18_noqc.yaml create mode 100644 parm/atm/obs/testing/amsua_n19_noqc.yaml diff --git a/parm/atm/obs/testing/amsua_metop-a.yaml b/parm/atm/obs/testing/amsua_metop-a.yaml index d2951e032..f7009b514 100644 --- a/parm/atm/obs/testing/amsua_metop-a.yaml +++ b/parm/atm/obs/testing/amsua_metop-a.yaml @@ -18,7 +18,7 @@ obs space: type: H5File obsfile: !ENV amsua_metop-a_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-15 + channels: &amsua_metop-a_channels 1-15 geovals: filename: !ENV amsua_metop-a_geoval_${CDATE}.nc4 obs bias: @@ -46,7 +46,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels flags: - name: CLWRetrievalReject initial value: false @@ -56,7 +56,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels flags: - name: HydrometeorCheckReject initial value: false @@ -66,7 +66,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels flags: - name: GrossCheckReject initial value: false @@ -76,7 +76,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels flags: - name: InterChannelCheckReject initial value: false @@ -153,13 +153,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/CLWMatchIndexMW - channels: *all_channels + channels: *amsua_metop-a_channels options: - channels: *all_channels + channels: *amsua_metop-a_channels clwobs_function: name: DerivedMetaData/CLWRetFromObs clwbkg_function: @@ -172,13 +172,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/ObsErrorModelRamp - channels: *all_channels + channels: *amsua_metop-a_channels options: - channels: *all_channels + channels: *amsua_metop-a_channels xvar: name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, @@ -198,29 +198,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/Innovation - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *amsua_metop-a_channels options: variables: - name: ObsValue/brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels - name: HofX/brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels coefs: [1, -1] # Step 1: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels action: name: assign error error function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-a_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -264,12 +264,12 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels test variables: - name: ObsFunction/HydrometeorCheckAMSUA - channels: *all_channels + channels: *amsua_metop-a_channels options: - channels: *all_channels + channels: *amsua_metop-a_channels obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -277,7 +277,7 @@ obs post filters: name: DerivedMetaData/CLWRetFromObs obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-a_channels maxvalue: 0.0 actions: - name: set @@ -289,84 +289,84 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/ObsErrorFactorTopoRad - channels: *all_channels + channels: *amsua_metop-a_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-a_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-a_channels # Step 7: Obs error inflation based on TOA transmittancec check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *all_channels + channels: *amsua_metop-a_channels options: - channels: *all_channels + channels: *amsua_metop-a_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-a_channels # Step 8: Observation error inflation based on surface jacobian check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *all_channels + channels: *amsua_metop-a_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-a_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_metop-a_channels # Step 9: Situation dependent check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/ObsErrorFactorSituDependMW - channels: *all_channels + channels: *amsua_metop-a_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-a_channels clwbkg_function: name: DerivedMetaData/CLWRetFromBkg clwobs_function: @@ -375,10 +375,10 @@ obs post filters: name: DerivedMetaData/SIRetFromObs clwmatchidx_function: name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_metop-a_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-a_channels obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -386,12 +386,12 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_metop-a_channels # Step 10: Gross check # Remove data if abs(Obs-HofX) > absolute threhold @@ -407,27 +407,27 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/ObsErrorBoundMW - channels: *all_channels + channels: *amsua_metop-a_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-a_channels obserr_bound_latitude: name: DerivedMetaData/ObsErrorFactorLat obserr_bound_transmittop: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-a_channels options: - channels: *all_channels + channels: *amsua_metop-a_channels obserr_bound_topo: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-a_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-a_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -436,10 +436,10 @@ obs post filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels function absolute threshold: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_metop-a_channels actions: - name: set flag: GrossCheckReject @@ -450,12 +450,12 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels test variables: - name: ObsFunction/InterChannelConsistencyCheck - channels: *all_channels + channels: *amsua_metop-a_channels options: - channels: *all_channels + channels: *amsua_metop-a_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, @@ -467,5 +467,5 @@ obs post filters: flag: InterChannelCheckReject ignore: rejected observations - name: reject -passedBenchmark: 93421 +passedBenchmark: 91979 diff --git a/parm/atm/obs/testing/amsua_metop-a_noqc.yaml b/parm/atm/obs/testing/amsua_metop-a_noqc.yaml new file mode 100644 index 000000000..a19c44833 --- /dev/null +++ b/parm/atm/obs/testing/amsua_metop-a_noqc.yaml @@ -0,0 +1,138 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-a + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: amsua_metop-a + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_metop-a_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_metop-a_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &amsua_metop-a_channels 1-15 +geovals: + filename: !ENV amsua_metop-a_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV amsua_metop-a_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-a_tlap !ENV amsua_metop-a_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_metop-a_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-a_channels + +passedBenchmark: 117872 +#vector ref: GsiHofXBc +#tolerance: 1.e-7 + diff --git a/parm/atm/obs/testing/amsua_metop-b.yaml b/parm/atm/obs/testing/amsua_metop-b.yaml index 3c666d8e9..ebd1f3999 100644 --- a/parm/atm/obs/testing/amsua_metop-b.yaml +++ b/parm/atm/obs/testing/amsua_metop-b.yaml @@ -18,7 +18,7 @@ obs space: type: H5File obsfile: !ENV amsua_metop-b_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-15 + channels: &amsua_metop-b_channels 1-15 geovals: filename: !ENV amsua_metop-b_geoval_${CDATE}.nc4 obs bias: @@ -46,7 +46,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels flags: - name: CLWRetrievalReject initial value: false @@ -56,7 +56,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels flags: - name: HydrometeorCheckReject initial value: false @@ -66,7 +66,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels flags: - name: GrossCheckReject initial value: false @@ -76,7 +76,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels flags: - name: InterChannelCheckReject initial value: false @@ -153,13 +153,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/CLWMatchIndexMW - channels: *all_channels + channels: *amsua_metop-b_channels options: - channels: *all_channels + channels: *amsua_metop-b_channels clwobs_function: name: DerivedMetaData/CLWRetFromObs clwbkg_function: @@ -172,13 +172,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/ObsErrorModelRamp - channels: *all_channels + channels: *amsua_metop-b_channels options: - channels: *all_channels + channels: *amsua_metop-b_channels xvar: name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, @@ -198,29 +198,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/Innovation - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *amsua_metop-b_channels options: variables: - name: ObsValue/brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels - name: HofX/brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels coefs: [1, -1] # Step 1: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels action: name: assign error error function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-b_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -264,12 +264,12 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels test variables: - name: ObsFunction/HydrometeorCheckAMSUA - channels: *all_channels + channels: *amsua_metop-b_channels options: - channels: *all_channels + channels: *amsua_metop-b_channels obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -277,7 +277,7 @@ obs post filters: name: DerivedMetaData/CLWRetFromObs obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-b_channels maxvalue: 0.0 actions: - name: set @@ -289,84 +289,84 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/ObsErrorFactorTopoRad - channels: *all_channels + channels: *amsua_metop-b_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-b_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-b_channels # Step 7: Obs error inflation based on TOA transmittancec check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *all_channels + channels: *amsua_metop-b_channels options: - channels: *all_channels + channels: *amsua_metop-b_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-b_channels # Step 8: Observation error inflation based on surface jacobian check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *all_channels + channels: *amsua_metop-b_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-b_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_metop-b_channels # Step 9: Situation dependent check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/ObsErrorFactorSituDependMW - channels: *all_channels + channels: *amsua_metop-b_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-b_channels clwbkg_function: name: DerivedMetaData/CLWRetFromBkg clwobs_function: @@ -375,10 +375,10 @@ obs post filters: name: DerivedMetaData/SIRetFromObs clwmatchidx_function: name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_metop-b_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-b_channels obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -386,12 +386,12 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_metop-b_channels # Step 10: Gross check # Remove data if abs(Obs-HofX) > absolute threhold @@ -407,27 +407,27 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/ObsErrorBoundMW - channels: *all_channels + channels: *amsua_metop-b_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-b_channels obserr_bound_latitude: name: DerivedMetaData/ObsErrorFactorLat obserr_bound_transmittop: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-b_channels options: - channels: *all_channels + channels: *amsua_metop-b_channels obserr_bound_topo: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-b_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-b_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -436,10 +436,10 @@ obs post filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels function absolute threshold: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_metop-b_channels actions: - name: set flag: GrossCheckReject @@ -450,12 +450,12 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels test variables: - name: ObsFunction/InterChannelConsistencyCheck - channels: *all_channels + channels: *amsua_metop-b_channels options: - channels: *all_channels + channels: *amsua_metop-b_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ -1, -1, -1, -1, -1, @@ -467,5 +467,5 @@ obs post filters: flag: InterChannelCheckReject ignore: rejected observations - name: reject -passedBenchmark: 71068 +passedBenchmark: 70590 diff --git a/parm/atm/obs/testing/amsua_metop-b_noqc.yaml b/parm/atm/obs/testing/amsua_metop-b_noqc.yaml new file mode 100644 index 000000000..adc4b9395 --- /dev/null +++ b/parm/atm/obs/testing/amsua_metop-b_noqc.yaml @@ -0,0 +1,137 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-b + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: amsua_metop-b + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_metop-b_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_metop-b_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &amsua_metop-b_channels 1-15 +geovals: + filename: !ENV amsua_metop-b_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV amsua_metop-b_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-b_tlap !ENV amsua_metop-b_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_metop-b_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-b_channels + +passedBenchmark: 124549 +#vector ref: GsiHofXBc +#tolerance: 1.e-7 diff --git a/parm/atm/obs/testing/amsua_metop-c.yaml b/parm/atm/obs/testing/amsua_metop-c.yaml index 0e9e82795..49871a8af 100644 --- a/parm/atm/obs/testing/amsua_metop-c.yaml +++ b/parm/atm/obs/testing/amsua_metop-c.yaml @@ -18,7 +18,7 @@ obs space: type: H5File obsfile: !ENV amsua_metop-c_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-15 + channels: &amsua_metop-c_channels 1-15 geovals: filename: !ENV amsua_metop-c_geoval_${CDATE}.nc4 obs bias: @@ -46,7 +46,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels flags: - name: CLWRetrievalReject initial value: false @@ -56,7 +56,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels flags: - name: HydrometeorCheckReject initial value: false @@ -66,7 +66,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels flags: - name: GrossCheckReject initial value: false @@ -76,7 +76,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels flags: - name: InterChannelCheckReject initial value: false @@ -153,13 +153,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/CLWMatchIndexMW - channels: *all_channels + channels: *amsua_metop-c_channels options: - channels: *all_channels + channels: *amsua_metop-c_channels clwobs_function: name: DerivedMetaData/CLWRetFromObs clwbkg_function: @@ -172,13 +172,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/ObsErrorModelRamp - channels: *all_channels + channels: *amsua_metop-c_channels options: - channels: *all_channels + channels: *amsua_metop-c_channels xvar: name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, @@ -198,29 +198,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/Innovation - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *amsua_metop-c_channels options: variables: - name: ObsValue/brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels - name: HofX/brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels coefs: [1, -1] # Step 1: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels action: name: assign error error function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-c_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -264,12 +264,12 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels test variables: - name: ObsFunction/HydrometeorCheckAMSUA - channels: *all_channels + channels: *amsua_metop-c_channels options: - channels: *all_channels + channels: *amsua_metop-c_channels obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -277,7 +277,7 @@ obs post filters: name: DerivedMetaData/CLWRetFromObs obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-c_channels maxvalue: 0.0 actions: - name: set @@ -289,84 +289,84 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/ObsErrorFactorTopoRad - channels: *all_channels + channels: *amsua_metop-c_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-c_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-c_channels # Step 7: Obs error inflation based on TOA transmittancec check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *all_channels + channels: *amsua_metop-c_channels options: - channels: *all_channels + channels: *amsua_metop-c_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-c_channels # Step 8: Observation error inflation based on surface jacobian check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *all_channels + channels: *amsua_metop-c_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-c_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_metop-c_channels # Step 9: Situation dependent check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/ObsErrorFactorSituDependMW - channels: *all_channels + channels: *amsua_metop-c_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-c_channels clwbkg_function: name: DerivedMetaData/CLWRetFromBkg clwobs_function: @@ -375,10 +375,10 @@ obs post filters: name: DerivedMetaData/SIRetFromObs clwmatchidx_function: name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_metop-c_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-c_channels obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -386,12 +386,12 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_metop-c_channels # Step 10: Gross check # Remove data if abs(Obs-HofX) > absolute threhold @@ -407,27 +407,27 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/ObsErrorBoundMW - channels: *all_channels + channels: *amsua_metop-c_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-c_channels obserr_bound_latitude: name: DerivedMetaData/ObsErrorFactorLat obserr_bound_transmittop: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-c_channels options: - channels: *all_channels + channels: *amsua_metop-c_channels obserr_bound_topo: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-c_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-c_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -436,10 +436,10 @@ obs post filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels function absolute threshold: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_metop-c_channels actions: - name: set flag: GrossCheckReject @@ -450,12 +450,12 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels test variables: - name: ObsFunction/InterChannelConsistencyCheck - channels: *all_channels + channels: *amsua_metop-c_channels options: - channels: *all_channels + channels: *amsua_metop-c_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, @@ -467,4 +467,4 @@ obs post filters: flag: InterChannelCheckReject ignore: rejected observations - name: reject -passedBenchmark: 113345 +passedBenchmark: 112377 diff --git a/parm/atm/obs/testing/amsua_metop-c_noqc.yaml b/parm/atm/obs/testing/amsua_metop-c_noqc.yaml new file mode 100644 index 000000000..885f3b38f --- /dev/null +++ b/parm/atm/obs/testing/amsua_metop-c_noqc.yaml @@ -0,0 +1,138 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-c + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: amsua_metop-c + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_metop-c_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_metop-c_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &amsua_metop-c_channels 1-15 +geovals: + filename: !ENV amsua_metop-c_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV amsua_metop-c_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-c_tlap !ENV amsua_metop-c_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_metop-c_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-c_channels + +passedBenchmark: 133601 +#vector ref: GsiHofXBc +#tolerance: 1.e-7 + diff --git a/parm/atm/obs/testing/amsua_n15.yaml b/parm/atm/obs/testing/amsua_n15.yaml index 973a2f26a..1afd7eb77 100644 --- a/parm/atm/obs/testing/amsua_n15.yaml +++ b/parm/atm/obs/testing/amsua_n15.yaml @@ -18,7 +18,7 @@ obs space: type: H5File obsfile: !ENV amsua_n15_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-15 + channels: &amsua_n15_channels 1-15 geovals: filename: !ENV amsua_n15_geoval_${CDATE}.nc4 obs bias: @@ -46,7 +46,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels flags: - name: CLWRetrievalReject initial value: false @@ -56,7 +56,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels flags: - name: HydrometeorCheckReject initial value: false @@ -66,7 +66,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels flags: - name: GrossCheckReject initial value: false @@ -76,7 +76,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels flags: - name: InterChannelCheckReject initial value: false @@ -153,13 +153,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/CLWMatchIndexMW - channels: *all_channels + channels: *amsua_n15_channels options: - channels: *all_channels + channels: *amsua_n15_channels clwobs_function: name: DerivedMetaData/CLWRetFromObs clwbkg_function: @@ -172,13 +172,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/ObsErrorModelRamp - channels: *all_channels + channels: *amsua_n15_channels options: - channels: *all_channels + channels: *amsua_n15_channels xvar: name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, @@ -198,29 +198,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/Innovation - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *amsua_n15_channels options: variables: - name: ObsValue/brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels - name: HofX/brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels coefs: [1, -1] # Step 1: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels action: name: assign error error function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n15_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -264,12 +264,12 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels test variables: - name: ObsFunction/HydrometeorCheckAMSUA - channels: *all_channels + channels: *amsua_n15_channels options: - channels: *all_channels + channels: *amsua_n15_channels obserr_clearsky: [ 3.000, 2.200, 2.000, 0.600, 0.300, 0.230, 0.250, 0.275, 0.340, 0.400, 0.600, 1.000, 1.500, 4.000, 3.500] @@ -277,7 +277,7 @@ obs post filters: name: DerivedMetaData/CLWRetFromObs obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n15_channels maxvalue: 0.0 actions: - name: set @@ -289,84 +289,84 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/ObsErrorFactorTopoRad - channels: *all_channels + channels: *amsua_n15_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n15_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n15_channels # Step 7: Obs error inflation based on TOA transmittancec check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *all_channels + channels: *amsua_n15_channels options: - channels: *all_channels + channels: *amsua_n15_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n15_channels # Step 8: Observation error inflation based on surface jacobian check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *all_channels + channels: *amsua_n15_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n15_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_n15_channels # Step 9: Situation dependent check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/ObsErrorFactorSituDependMW - channels: *all_channels + channels: *amsua_n15_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n15_channels clwbkg_function: name: DerivedMetaData/CLWRetFromBkg clwobs_function: @@ -375,10 +375,10 @@ obs post filters: name: DerivedMetaData/SIRetFromObs clwmatchidx_function: name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_n15_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n15_channels obserr_clearsky: [ 3.000, 2.200, 2.000, 0.600, 0.300, 0.230, 0.250, 0.275, 0.340, 0.400, 0.600, 1.000, 1.500, 4.000, 3.500] @@ -386,12 +386,12 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_n15_channels # Step 10: Gross check # Remove data if abs(Obs-HofX) > absolute threhold @@ -407,27 +407,27 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/ObsErrorBoundMW - channels: *all_channels + channels: *amsua_n15_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n15_channels obserr_bound_latitude: name: DerivedMetaData/ObsErrorFactorLat obserr_bound_transmittop: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n15_channels options: - channels: *all_channels + channels: *amsua_n15_channels obserr_bound_topo: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n15_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n15_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -436,10 +436,10 @@ obs post filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels function absolute threshold: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_n15_channels actions: - name: set flag: GrossCheckReject @@ -450,12 +450,12 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels test variables: - name: ObsFunction/InterChannelConsistencyCheck - channels: *all_channels + channels: *amsua_n15_channels options: - channels: *all_channels + channels: *amsua_n15_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, @@ -467,4 +467,4 @@ obs post filters: flag: InterChannelCheckReject ignore: rejected observations - name: reject -passedBenchmark: 91792 +passedBenchmark: 85704 diff --git a/parm/atm/obs/testing/amsua_n15_noqc.yaml b/parm/atm/obs/testing/amsua_n15_noqc.yaml new file mode 100644 index 000000000..a16d1aed7 --- /dev/null +++ b/parm/atm/obs/testing/amsua_n15_noqc.yaml @@ -0,0 +1,137 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n15 + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: amsua_n15 + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_n15_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_n15_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &amsua_n15_channels 1-15 +geovals: + filename: !ENV amsua_n15_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV amsua_n15_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n15_tlap !ENV amsua_n15_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_n15_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 18.000] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n15_channels + +passedBenchmark: 116256 +#vector ref: GsiHofXBc +#tolerance: 1.e-7 diff --git a/parm/atm/obs/testing/amsua_n18.yaml b/parm/atm/obs/testing/amsua_n18.yaml index d11eba99b..9dff0330c 100644 --- a/parm/atm/obs/testing/amsua_n18.yaml +++ b/parm/atm/obs/testing/amsua_n18.yaml @@ -18,7 +18,7 @@ obs space: type: H5File obsfile: !ENV amsua_n18_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-15 + channels: &amsua_n18_channels 1-15 geovals: filename: !ENV amsua_n18_geoval_${CDATE}.nc4 obs bias: @@ -46,7 +46,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels flags: - name: CLWRetrievalReject initial value: false @@ -56,7 +56,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels flags: - name: HydrometeorCheckReject initial value: false @@ -66,7 +66,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels flags: - name: GrossCheckReject initial value: false @@ -76,7 +76,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels flags: - name: InterChannelCheckReject initial value: false @@ -153,13 +153,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/CLWMatchIndexMW - channels: *all_channels + channels: *amsua_n18_channels options: - channels: *all_channels + channels: *amsua_n18_channels clwobs_function: name: DerivedMetaData/CLWRetFromObs clwbkg_function: @@ -172,13 +172,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/ObsErrorModelRamp - channels: *all_channels + channels: *amsua_n18_channels options: - channels: *all_channels + channels: *amsua_n18_channels xvar: name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, @@ -198,29 +198,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/Innovation - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *amsua_n18_channels options: variables: - name: ObsValue/brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels - name: HofX/brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels coefs: [1, -1] # Step 1: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels action: name: assign error error function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n18_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -264,12 +264,12 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels test variables: - name: ObsFunction/HydrometeorCheckAMSUA - channels: *all_channels + channels: *amsua_n18_channels options: - channels: *all_channels + channels: *amsua_n18_channels obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -277,7 +277,7 @@ obs post filters: name: DerivedMetaData/CLWRetFromObs obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n18_channels maxvalue: 0.0 actions: - name: set @@ -289,84 +289,84 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/ObsErrorFactorTopoRad - channels: *all_channels + channels: *amsua_n18_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n18_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n18_channels # Step 7: Obs error inflation based on TOA transmittancec check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *all_channels + channels: *amsua_n18_channels options: - channels: *all_channels + channels: *amsua_n18_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n18_channels # Step 8: Observation error inflation based on surface jacobian check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *all_channels + channels: *amsua_n18_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n18_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_n18_channels # Step 9: Situation dependent check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/ObsErrorFactorSituDependMW - channels: *all_channels + channels: *amsua_n18_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n18_channels clwbkg_function: name: DerivedMetaData/CLWRetFromBkg clwobs_function: @@ -375,10 +375,10 @@ obs post filters: name: DerivedMetaData/SIRetFromObs clwmatchidx_function: name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_n18_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n18_channels obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -386,12 +386,12 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_n18_channels # Step 10: Gross check # Remove data if abs(Obs-HofX) > absolute threhold @@ -407,27 +407,27 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/ObsErrorBoundMW - channels: *all_channels + channels: *amsua_n18_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n18_channels obserr_bound_latitude: name: DerivedMetaData/ObsErrorFactorLat obserr_bound_transmittop: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n18_channels options: - channels: *all_channels + channels: *amsua_n18_channels obserr_bound_topo: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n18_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n18_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -436,10 +436,10 @@ obs post filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels function absolute threshold: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_n18_channels actions: - name: set flag: GrossCheckReject @@ -450,12 +450,12 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels test variables: - name: ObsFunction/InterChannelConsistencyCheck - channels: *all_channels + channels: *amsua_n18_channels options: - channels: *all_channels + channels: *amsua_n18_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, -1, @@ -467,5 +467,5 @@ obs post filters: flag: InterChannelCheckReject ignore: rejected observations - name: reject -passedBenchmark: 116999 +passedBenchmark: 115823 diff --git a/parm/atm/obs/testing/amsua_n18_noqc.yaml b/parm/atm/obs/testing/amsua_n18_noqc.yaml new file mode 100644 index 000000000..5bee14388 --- /dev/null +++ b/parm/atm/obs/testing/amsua_n18_noqc.yaml @@ -0,0 +1,137 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n18 + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: amsua_n18 + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_n18_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_n18_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &amsua_n18_channels 1-15 +geovals: + filename: !ENV amsua_n18_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV amsua_n18_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n18_tlap !ENV amsua_n18_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_n18_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n18_channels + +passedBenchmark: 137130 +#vector ref: GsiHofXBc +#tolerance: 1.e-7 diff --git a/parm/atm/obs/testing/amsua_n19.yaml b/parm/atm/obs/testing/amsua_n19.yaml index f91b045c1..9eff67593 100644 --- a/parm/atm/obs/testing/amsua_n19.yaml +++ b/parm/atm/obs/testing/amsua_n19.yaml @@ -18,7 +18,7 @@ obs space: type: H5File obsfile: !ENV amsua_n19_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-15 + channels: &amsua_n19_channels 1-15 geovals: filename: !ENV amsua_n19_geoval_${CDATE}.nc4 obs bias: @@ -46,7 +46,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels flags: - name: CLWRetrievalReject initial value: false @@ -56,7 +56,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels flags: - name: HydrometeorCheckReject initial value: false @@ -66,7 +66,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels flags: - name: GrossCheckReject initial value: false @@ -76,7 +76,7 @@ obs post filters: - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels flags: - name: InterChannelCheckReject initial value: false @@ -153,13 +153,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/CLWMatchIndexMW - channels: *all_channels + channels: *amsua_n19_channels options: - channels: *all_channels + channels: *amsua_n19_channels clwobs_function: name: DerivedMetaData/CLWRetFromObs clwbkg_function: @@ -172,13 +172,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/ObsErrorModelRamp - channels: *all_channels + channels: *amsua_n19_channels options: - channels: *all_channels + channels: *amsua_n19_channels xvar: name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, @@ -198,29 +198,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/Innovation - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *amsua_n19_channels options: variables: - name: ObsValue/brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels - name: HofX/brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels coefs: [1, -1] # Step 1: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels action: name: assign error error function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n19_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -264,12 +264,12 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels test variables: - name: ObsFunction/HydrometeorCheckAMSUA - channels: *all_channels + channels: *amsua_n19_channels options: - channels: *all_channels + channels: *amsua_n19_channels obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -277,7 +277,7 @@ obs post filters: name: DerivedMetaData/CLWRetFromObs obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n19_channels maxvalue: 0.0 actions: - name: set @@ -289,84 +289,84 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/ObsErrorFactorTopoRad - channels: *all_channels + channels: *amsua_n19_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n19_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n19_channels # Step 7: Obs error inflation based on TOA transmittancec check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *all_channels + channels: *amsua_n19_channels options: - channels: *all_channels + channels: *amsua_n19_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n19_channels # Step 8: Observation error inflation based on surface jacobian check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *all_channels + channels: *amsua_n19_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n19_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_n19_channels # Step 9: Situation dependent check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/ObsErrorFactorSituDependMW - channels: *all_channels + channels: *amsua_n19_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n19_channels clwbkg_function: name: DerivedMetaData/CLWRetFromBkg clwobs_function: @@ -375,10 +375,10 @@ obs post filters: name: DerivedMetaData/SIRetFromObs clwmatchidx_function: name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_n19_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n19_channels obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -386,12 +386,12 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_n19_channels # Step 10: Gross check # Remove data if abs(Obs-HofX) > absolute threhold @@ -407,27 +407,27 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/ObsErrorBoundMW - channels: *all_channels + channels: *amsua_n19_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n19_channels obserr_bound_latitude: name: DerivedMetaData/ObsErrorFactorLat obserr_bound_transmittop: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n19_channels options: - channels: *all_channels + channels: *amsua_n19_channels obserr_bound_topo: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n19_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n19_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -436,10 +436,10 @@ obs post filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels function absolute threshold: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_n19_channels actions: - name: set flag: GrossCheckReject @@ -450,12 +450,12 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels test variables: - name: ObsFunction/InterChannelConsistencyCheck - channels: *all_channels + channels: *amsua_n19_channels options: - channels: *all_channels + channels: *amsua_n19_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, @@ -467,4 +467,4 @@ obs post filters: flag: InterChannelCheckReject ignore: rejected observations - name: reject -passedBenchmark: 107747 +passedBenchmark: 106461 diff --git a/parm/atm/obs/testing/amsua_n19_noqc.yaml b/parm/atm/obs/testing/amsua_n19_noqc.yaml new file mode 100644 index 000000000..b8ddd2b86 --- /dev/null +++ b/parm/atm/obs/testing/amsua_n19_noqc.yaml @@ -0,0 +1,137 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n19 + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: amsua_n19 + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_n19_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_n19_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &amsua_n19_channels 1-15 +geovals: + filename: !ENV amsua_n19_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV amsua_n19_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n19_tlap !ENV amsua_n19_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_n19_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_n19_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n19_channels + +passedBenchmark: 136652 +#vector ref: GsiHofXBc +#tolerance: 1.e-7 From f572a2b210b2eae6b38b18b5c801b294ed0fa812 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Mon, 20 Nov 2023 23:44:20 -0600 Subject: [PATCH 28/58] For end-to-end configuration AMSU-A YAML files: update n19 and add n18, n15, metop-b and metop-c --- parm/atm/obs/config/amsua_metop-a.yaml | 484 +++++++++++++++++++++++ parm/atm/obs/config/amsua_metop-b.yaml | 484 +++++++++++++++++++++++ parm/atm/obs/config/amsua_metop-c.yaml | 483 +++++++++++++++++++++++ parm/atm/obs/config/amsua_n15.yaml | 483 +++++++++++++++++++++++ parm/atm/obs/config/amsua_n18.yaml | 484 +++++++++++++++++++++++ parm/atm/obs/config/amsua_n19.yaml | 526 +++++++++++++++---------- 6 files changed, 2739 insertions(+), 205 deletions(-) create mode 100644 parm/atm/obs/config/amsua_metop-a.yaml create mode 100644 parm/atm/obs/config/amsua_metop-b.yaml create mode 100644 parm/atm/obs/config/amsua_metop-c.yaml create mode 100644 parm/atm/obs/config/amsua_n15.yaml create mode 100644 parm/atm/obs/config/amsua_n18.yaml diff --git a/parm/atm/obs/config/amsua_metop-a.yaml b/parm/atm/obs/config/amsua_metop-a.yaml new file mode 100644 index 000000000..d9147b7de --- /dev/null +++ b/parm/atm/obs/config/amsua_metop-a.yaml @@ -0,0 +1,484 @@ +obs space: + name: amsua_metop-a + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)amsua_metop-a.{{ current_cycle | to_YMDH }}.nc4 + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_amsua_metop-a_{{ current_cycle | to_YMDH }}.nc4 + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &amsua_metop-a_channels 1-15 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: amsua_metop-a + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)amsua_metop-a.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_metop-a.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-a_tlapse $(DATA)/obs/$(GPREFIX)amsua_metop-a.tlapse.txt + - name: lapse_rate + tlapse: *amsua_metop-a_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)amsua_metop-a.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)amsua_metop-a.satbias_cov.nc4 + +obs post filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + flags: + - name: CLWRetrievalReject + initial value: false + force reinitialization: true + +# Diagnostic flag for hydrometeor check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + flags: + - name: HydrometeorCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for gross check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + flags: + - name: GrossCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for inter-channel consistency check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + flags: + - name: InterChannelCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *amsua_metop-a_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *amsua_metop-a_channels + - name: HofX/brightnessTemperature + channels: *amsua_metop-a_channels + coefs: [1, -1] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-a_channels + +# Step 2: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 3: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-a_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheckReject + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *amsua_metop-a_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-a_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-a_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-a_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *amsua_metop-a_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-a_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_metop-a_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *amsua_metop-a_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-a_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *amsua_metop-a_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-a_channels + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_metop-a_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *amsua_metop-a_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-a_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-a_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-a_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_metop-a_channels + actions: + - name: set + flag: GrossCheckReject + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, 1, 1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelCheckReject + ignore: rejected observations + - name: reject + diff --git a/parm/atm/obs/config/amsua_metop-b.yaml b/parm/atm/obs/config/amsua_metop-b.yaml new file mode 100644 index 000000000..c817015cf --- /dev/null +++ b/parm/atm/obs/config/amsua_metop-b.yaml @@ -0,0 +1,484 @@ +obs space: + name: amsua_metop-b + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)amsua_metop-b.{{ current_cycle | to_YMDH }}.nc4 + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_amsua_metop-b_{{ current_cycle | to_YMDH }}.nc4 + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &amsua_metop-b_channels 1-15 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: amsua_metop-b + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)amsua_metop-b.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_metop-b.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-b_tlapse $(DATA)/obs/$(GPREFIX)amsua_metop-b.tlapse.txt + - name: lapse_rate + tlapse: *amsua_metop-b_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)amsua_metop-b.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)amsua_metop-b.satbias_cov.nc4 + +obs post filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + flags: + - name: CLWRetrievalReject + initial value: false + force reinitialization: true + +# Diagnostic flag for hydrometeor check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + flags: + - name: HydrometeorCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for gross check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + flags: + - name: GrossCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for inter-channel consistency check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + flags: + - name: InterChannelCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *amsua_metop-b_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *amsua_metop-b_channels + - name: HofX/brightnessTemperature + channels: *amsua_metop-b_channels + coefs: [1, -1] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-b_channels + +# Step 2: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 3: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-b_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheckReject + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *amsua_metop-b_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-b_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-b_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-b_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *amsua_metop-b_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-b_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_metop-b_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *amsua_metop-b_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-b_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *amsua_metop-b_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-b_channels + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_metop-b_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *amsua_metop-b_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-b_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-b_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-b_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_metop-b_channels + actions: + - name: set + flag: GrossCheckReject + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, + 1, 1, 1, 1, -1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelCheckReject + ignore: rejected observations + - name: reject + diff --git a/parm/atm/obs/config/amsua_metop-c.yaml b/parm/atm/obs/config/amsua_metop-c.yaml new file mode 100644 index 000000000..bb3d2e511 --- /dev/null +++ b/parm/atm/obs/config/amsua_metop-c.yaml @@ -0,0 +1,483 @@ +obs space: + name: amsua_metop-c + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)amsua_metop-c.{{ current_cycle | to_YMDH }}.nc4 + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_amsua_metop-c_{{ current_cycle | to_YMDH }}.nc4 + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &amsua_metop-c_channels 1-15 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: amsua_metop-c + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)amsua_metop-c.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_metop-c.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-c_tlapse $(DATA)/obs/$(GPREFIX)amsua_metop-c.tlapse.txt + - name: lapse_rate + tlapse: *amsua_metop-c_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)amsua_metop-c.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)amsua_metop-c.satbias_cov.nc4 + +obs post filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + flags: + - name: CLWRetrievalReject + initial value: false + force reinitialization: true + +# Diagnostic flag for hydrometeor check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + flags: + - name: HydrometeorCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for gross check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + flags: + - name: GrossCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for inter-channel consistency check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + flags: + - name: InterChannelCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *amsua_metop-c_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *amsua_metop-c_channels + - name: HofX/brightnessTemperature + channels: *amsua_metop-c_channels + coefs: [1, -1] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-c_channels + +# Step 2: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 3: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-c_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheckReject + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *amsua_metop-c_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-c_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-c_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-c_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *amsua_metop-c_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-c_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_metop-c_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *amsua_metop-c_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-c_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *amsua_metop-c_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-c_channels + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_metop-c_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *amsua_metop-c_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-c_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-c_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-c_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_metop-c_channels + actions: + - name: set + flag: GrossCheckReject + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelCheckReject + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/amsua_n15.yaml b/parm/atm/obs/config/amsua_n15.yaml new file mode 100644 index 000000000..b32c50ca7 --- /dev/null +++ b/parm/atm/obs/config/amsua_n15.yaml @@ -0,0 +1,483 @@ +obs space: + name: amsua_n15 + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)amsua_n15.{{ current_cycle | to_YMDH }}.nc4 + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_amsua_n15_{{ current_cycle | to_YMDH }}.nc4 + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &amsua_n15_channels 1-15 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: amsua_n15 + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)amsua_n15.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n15.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n15_tlapse $(DATA)/obs/$(GPREFIX)amsua_n15.tlapse.txt + - name: lapse_rate + tlapse: *amsua_n15_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)amsua_n15.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)amsua_n15.satbias_cov.nc4 + +obs post filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + flags: + - name: CLWRetrievalReject + initial value: false + force reinitialization: true + +# Diagnostic flag for hydrometeor check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + flags: + - name: HydrometeorCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for gross check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + flags: + - name: GrossCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for inter-channel consistency check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + flags: + - name: InterChannelCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *amsua_n15_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *amsua_n15_channels + - name: HofX/brightnessTemperature + channels: *amsua_n15_channels + coefs: [1, -1] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n15_channels + +# Step 2: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 3: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + obserr_clearsky: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n15_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheckReject + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *amsua_n15_channels + options: + sensor: *Sensor_ID + channels: *amsua_n15_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n15_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n15_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *amsua_n15_channels + options: + sensor: *Sensor_ID + channels: *amsua_n15_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_n15_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *amsua_n15_channels + options: + sensor: *Sensor_ID + channels: *amsua_n15_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *amsua_n15_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n15_channels + obserr_clearsky: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_n15_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *amsua_n15_channels + options: + sensor: *Sensor_ID + channels: *amsua_n15_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n15_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n15_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_n15_channels + actions: + - name: set + flag: GrossCheckReject + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, 1, + -1, 1, 1, 1, 1, + -1, 1, 1, -1, 1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelCheckReject + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/amsua_n18.yaml b/parm/atm/obs/config/amsua_n18.yaml new file mode 100644 index 000000000..574b50c2e --- /dev/null +++ b/parm/atm/obs/config/amsua_n18.yaml @@ -0,0 +1,484 @@ +obs space: + name: amsua_n18 + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)amsua_n18.{{ current_cycle | to_YMDH }}.nc4 + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_amsua_n18_{{ current_cycle | to_YMDH }}.nc4 + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &amsua_n18_channels 1-15 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: amsua_n18 + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)amsua_n18.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n18.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n18_tlapse $(DATA)/obs/$(GPREFIX)amsua_n18.tlapse.txt + - name: lapse_rate + tlapse: *amsua_n18_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)amsua_n18.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)amsua_n18.satbias_cov.nc4 + +obs post filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + flags: + - name: CLWRetrievalReject + initial value: false + force reinitialization: true + +# Diagnostic flag for hydrometeor check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + flags: + - name: HydrometeorCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for gross check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + flags: + - name: GrossCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for inter-channel consistency check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + flags: + - name: InterChannelCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *amsua_n18_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *amsua_n18_channels + - name: HofX/brightnessTemperature + channels: *amsua_n18_channels + coefs: [1, -1] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n18_channels + +# Step 2: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 3: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n18_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheckReject + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *amsua_n18_channels + options: + sensor: *Sensor_ID + channels: *amsua_n18_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n18_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n18_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *amsua_n18_channels + options: + sensor: *Sensor_ID + channels: *amsua_n18_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_n18_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *amsua_n18_channels + options: + sensor: *Sensor_ID + channels: *amsua_n18_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *amsua_n18_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n18_channels + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_n18_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *amsua_n18_channels + options: + sensor: *Sensor_ID + channels: *amsua_n18_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n18_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n18_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_n18_channels + actions: + - name: set + flag: GrossCheckReject + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, -1, + 1, 1, -1, -1, 1, + 1, 1, 1, 1, 1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelCheckReject + ignore: rejected observations + - name: reject + diff --git a/parm/atm/obs/config/amsua_n19.yaml b/parm/atm/obs/config/amsua_n19.yaml index df6c38528..a185d2485 100644 --- a/parm/atm/obs/config/amsua_n19.yaml +++ b/parm/atm/obs/config/amsua_n19.yaml @@ -12,6 +12,7 @@ obs space: max pool size: 1 simulated variables: [brightnessTemperature] channels: &amsua_n19_channels 1-15 + obs operator: name: CRTM Absorbers: [H2O,O3] @@ -21,6 +22,7 @@ obs operator: Sensor_ID: amsua_n19 EndianType: little_endian CoefficientPath: $(DATA)/crtm/ + obs bias: input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias.nc4 output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias.nc4 @@ -51,24 +53,148 @@ obs bias: ratio: 1.1 ratio for small dataset: 2.0 output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias_cov.nc4 -obs filters: -- filter: BlackList - filter variables: - - name: brightnessTemperature + +obs post filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature channels: *amsua_n19_channels - action: - name: assign error - error function: + flags: + - name: CLWRetrievalReject + initial value: false + force reinitialization: true + +# Diagnostic flag for hydrometeor check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_channels + flags: + - name: HydrometeorCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for gross check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_channels + flags: + - name: GrossCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for inter-channel consistency check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_channels + flags: + - name: InterChannelCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *amsua_n19_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_n19_channels + type: float + function: name: ObsFunction/ObsErrorModelRamp channels: *amsua_n19_channels options: channels: *amsua_n19_channels xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] + name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, 0.100, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.030] @@ -77,39 +203,78 @@ obs filters: 0.000, 0.000, 0.000, 0.000, 0.200] err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] + 0.400, 0.550, 0.800, 4.000, 3.500] err1: [20.000, 18.000, 12.000, 3.000, 0.500, 0.300, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 18.000] -# CLW Retrieval Check + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *amsua_n19_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *amsua_n19_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *amsua_n19_channels + - name: HofX/brightnessTemperature + channels: *amsua_n19_channels + coefs: [1, -1] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n19_channels + +# Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check filter variables: - name: brightnessTemperature channels: 1-6, 15 test variables: - - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] + - name: DerivedMetaData/CLWRetFromObs maxvalue: 999.0 - action: - name: reject -# CLW Retrieval Check + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 3: CLW Retrieval Check (background_based) - filter: Bounds Check filter variables: - name: brightnessTemperature channels: 1-6, 15 test variables: - - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] + - name: DerivedMetaData/CLWRetFromBkg maxvalue: 999.0 - action: - name: reject -# Hydrometeor Check (cloud/precipitation affected chanels) + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -121,247 +286,198 @@ obs filters: channels: *amsua_n19_channels obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] + 0.400, 0.550, 0.800, 4.000, 3.500] clwret_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] + name: DerivedMetaData/CLWRetFromObs obserr_function: - name: ObsFunction/ObsErrorModelRamp + name: DerivedMetaData/InitialObsError channels: *amsua_n19_channels - options: - channels: *amsua_n19_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, - 0.100, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.030] - x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, - 1.500, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.200] - err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, - 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] - err1: [20.000, 18.000, 12.000, 3.000, 0.500, - 0.300, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 18.000] maxvalue: 0.0 - action: - name: reject -# Topography check -- filter: BlackList - filter variables: - - name: brightnessTemperature + actions: + - name: set + flag: HydrometeorCheckReject + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_n19_channels - action: - name: inflate error - inflation variable: + type: float + function: name: ObsFunction/ObsErrorFactorTopoRad channels: *amsua_n19_channels options: - sensor: amsua_n19 + sensor: *Sensor_ID channels: *amsua_n19_channels -# Transmittnace Top Check -- filter: BlackList + +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *amsua_n19_channels action: name: inflate error inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n19_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n19_channels + type: float + function: name: ObsFunction/ObsErrorFactorTransmitTopRad channels: *amsua_n19_channels options: - sensor: amsua_n19 channels: *amsua_n19_channels -# Surface Jacobian check -- filter: BlackList + +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *amsua_n19_channels action: name: inflate error inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n19_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_n19_channels + type: float + function: name: ObsFunction/ObsErrorFactorSurfJacobianRad channels: *amsua_n19_channels options: - sensor: amsua_n19 + sensor: *Sensor_ID channels: *amsua_n19_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] -# Situation dependent Check -- filter: BlackList + +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *amsua_n19_channels action: name: inflate error inflation variable: - name: ObsFunction/ObsErrorFactorSituDependMW + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_n19_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_n19_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW channels: *amsua_n19_channels options: - sensor: amsua_n19 + sensor: *Sensor_ID channels: *amsua_n19_channels - clwobs_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] clwbkg_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - bias_application: HofX + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs scatobs_function: - name: ObsFunction/SCATRetMW - options: - scatret_ch238: 1 - scatret_ch314: 2 - scatret_ch890: 15 - scatret_types: [ObsValue] - bias_application: HofX + name: DerivedMetaData/SIRetFromObs clwmatchidx_function: - name: ObsFunction/CLWMatchIndexMW + name: DerivedMetaData/CLWMatchIndex channels: *amsua_n19_channels - options: - channels: *amsua_n19_channels - clwobs_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - clwbkg_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - bias_application: HofX - clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, - 0.100, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.030] obserr_function: - name: ObsFunction/ObsErrorModelRamp + name: DerivedMetaData/InitialObsError channels: *amsua_n19_channels - options: - channels: *amsua_n19_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, - 0.100, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.030] - x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, - 1.500, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.200] - err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, - 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] - err1: [20.000, 18.000, 12.000, 3.000, 0.500, - 0.300, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 18.000] obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] -# Gross check -- filter: Background Check + 0.400, 0.550, 0.800, 4.000, 3.500] + +- filter: Perform Action filter variables: - name: brightnessTemperature + channels: *amsua_n19_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_n19_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound channels: *amsua_n19_channels - function absolute threshold: - - name: ObsFunction/ObsErrorBoundMW - channels: *amsua_n19_channels - options: - sensor: amsua_n19 + type: float + function: + name: ObsFunction/ObsErrorBoundMW channels: *amsua_n19_channels - obserr_bound_latitude: - name: ObsFunction/ObsErrorFactorLatRad - options: - latitude_parameters: [25.0, 0.25, 0.04, 3.0] - obserr_bound_transmittop: - name: ObsFunction/ObsErrorFactorTransmitTopRad + options: + sensor: *Sensor_ID channels: *amsua_n19_channels - options: + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_n19_channels - obserr_bound_topo: - name: ObsFunction/ObsErrorFactorTopoRad - channels: *amsua_n19_channels - options: + options: + channels: *amsua_n19_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_n19_channels - sensor: amsua_n19 - obserr_function: - name: ObsFunction/ObsErrorModelRamp - channels: *amsua_n19_channels - options: + obserr_function: + name: DerivedMetaData/InitialObsError channels: *amsua_n19_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - bias_application: HofX - x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, - 0.100, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.030] - x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, - 1.500, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.200] - err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, - 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] - err1: [20.000, 18.000, 12.000, 3.000, 0.500, - 0.300, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 18.000] - obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, - 2.5, 3.5, 4.5, 4.5, 4.5] - action: - name: reject -# Inter-channel check -- filter: Bounds Check + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check filter variables: - name: brightnessTemperature channels: *amsua_n19_channels - test variables: - - name: ObsFunction/InterChannelConsistencyCheck + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound channels: *amsua_n19_channels - options: - channels: *amsua_n19_channels - sensor: amsua_n19 - use_flag: [ 1, 1, 1, 1, 1, - 1, -1, -1, 1, 1, - 1, 1, 1, -1, 1 ] - maxvalue: 1.0e-12 - action: - name: reject -# Useflag check + actions: + - name: set + flag: GrossCheckReject + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check - filter: Bounds Check filter variables: - name: brightnessTemperature channels: *amsua_n19_channels test variables: - - name: ObsFunction/ChannelUseflagCheckRad + - name: ObsFunction/InterChannelConsistencyCheck channels: *amsua_n19_channels options: - sensor: amaua_n19 channels: *amsua_n19_channels + use passive_bc: true + sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, - 1, 1, 1, -1, 1 ] - minvalue: 1.0e-12 - action: - name: reject + 1, 1, 1, 1, 1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelCheckReject + ignore: rejected observations + - name: reject From 179d8799bcb24c90518662721c494242c62c1d59 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Mon, 20 Nov 2023 23:46:06 -0600 Subject: [PATCH 29/58] Add various platforms for amsua for 3dvar test --- parm/atm/obs/lists/gdas_prototype_3d.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml b/parm/atm/obs/lists/gdas_prototype_3d.yaml index e8f2d006c..82cf45285 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml @@ -1,5 +1,10 @@ observers: ##- !INC ${OBS_YAML_DIR}/amsua_n19.yaml +##- !INC ${OBS_YAML_DIR}/amsua_n18.yaml +##- !INC ${OBS_YAML_DIR}/amsua_n15.yaml +##- !INC ${OBS_YAML_DIR}/amsua_metop-a.yaml +##- !INC ${OBS_YAML_DIR}/amsua_metop-b.yaml +##- !INC ${OBS_YAML_DIR}/amsua_metop-c.yaml ##- !INC ${OBS_YAML_DIR}/sondes_prepbufr.yaml ##- !INC ${OBS_YAML_DIR}/atms_n20.yaml ##- !INC ${OBS_YAML_DIR}/atms_npp.yaml @@ -15,4 +20,4 @@ observers: ##- !INC ${OBS_YAML_DIR}/cris-fsr_npp.yaml ##- !INC ${OBS_YAML_DIR}/sfc.yaml ##- !INC ${OBS_YAML_DIR}/sfcship.yaml -##- !INC ${OBS_YAML_DIR}/gnssro.yaml \ No newline at end of file +##- !INC ${OBS_YAML_DIR}/gnssro.yaml From 92e99a4098cbfcd28724832b0a2d1cb98b6234af Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 21 Nov 2023 10:10:35 -0600 Subject: [PATCH 30/58] Bug fixes: define Sensor_ID, modify obsfile names to follow naming convention. --- parm/atm/obs/config/amsua_metop-a.yaml | 6 +++--- parm/atm/obs/config/amsua_metop-b.yaml | 6 +++--- parm/atm/obs/config/amsua_metop-c.yaml | 6 +++--- parm/atm/obs/config/amsua_n15.yaml | 6 +++--- parm/atm/obs/config/amsua_n18.yaml | 6 +++--- parm/atm/obs/config/amsua_n19.yaml | 6 +++--- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/parm/atm/obs/config/amsua_metop-a.yaml b/parm/atm/obs/config/amsua_metop-a.yaml index d9147b7de..094142c14 100644 --- a/parm/atm/obs/config/amsua_metop-a.yaml +++ b/parm/atm/obs/config/amsua_metop-a.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)amsua_metop-a.{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)amsua_metop-a.tm00.nc obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_amsua_metop-a_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_amsua_metop-a_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [brightnessTemperature] @@ -19,7 +19,7 @@ obs operator: Clouds: [Water, Ice] Cloud_Fraction: 1.0 obs options: - Sensor_ID: amsua_metop-a + Sensor_ID: &Sensor_ID amsua_metop-a EndianType: little_endian CoefficientPath: $(DATA)/crtm/ diff --git a/parm/atm/obs/config/amsua_metop-b.yaml b/parm/atm/obs/config/amsua_metop-b.yaml index c817015cf..b6c0509d2 100644 --- a/parm/atm/obs/config/amsua_metop-b.yaml +++ b/parm/atm/obs/config/amsua_metop-b.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)amsua_metop-b.{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)amsua_metop-b.tm00.nc obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_amsua_metop-b_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_amsua_metop-b_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [brightnessTemperature] @@ -19,7 +19,7 @@ obs operator: Clouds: [Water, Ice] Cloud_Fraction: 1.0 obs options: - Sensor_ID: amsua_metop-b + Sensor_ID: &Sensor_ID amsua_metop-b EndianType: little_endian CoefficientPath: $(DATA)/crtm/ diff --git a/parm/atm/obs/config/amsua_metop-c.yaml b/parm/atm/obs/config/amsua_metop-c.yaml index bb3d2e511..1c4e1acca 100644 --- a/parm/atm/obs/config/amsua_metop-c.yaml +++ b/parm/atm/obs/config/amsua_metop-c.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)amsua_metop-c.{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)amsua_metop-c.tm00.nc obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_amsua_metop-c_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_amsua_metop-c_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [brightnessTemperature] @@ -19,7 +19,7 @@ obs operator: Clouds: [Water, Ice] Cloud_Fraction: 1.0 obs options: - Sensor_ID: amsua_metop-c + Sensor_ID: &Sensor_ID amsua_metop-c EndianType: little_endian CoefficientPath: $(DATA)/crtm/ diff --git a/parm/atm/obs/config/amsua_n15.yaml b/parm/atm/obs/config/amsua_n15.yaml index b32c50ca7..324f1ed96 100644 --- a/parm/atm/obs/config/amsua_n15.yaml +++ b/parm/atm/obs/config/amsua_n15.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)amsua_n15.{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)amsua_n15.tm00.nc obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_amsua_n15_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_amsua_n15_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [brightnessTemperature] @@ -19,7 +19,7 @@ obs operator: Clouds: [Water, Ice] Cloud_Fraction: 1.0 obs options: - Sensor_ID: amsua_n15 + Sensor_ID: &Sensor_ID amsua_n15 EndianType: little_endian CoefficientPath: $(DATA)/crtm/ diff --git a/parm/atm/obs/config/amsua_n18.yaml b/parm/atm/obs/config/amsua_n18.yaml index 574b50c2e..51648d34a 100644 --- a/parm/atm/obs/config/amsua_n18.yaml +++ b/parm/atm/obs/config/amsua_n18.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)amsua_n18.{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)amsua_n18.tm00.nc obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_amsua_n18_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_amsua_n18_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [brightnessTemperature] @@ -19,7 +19,7 @@ obs operator: Clouds: [Water, Ice] Cloud_Fraction: 1.0 obs options: - Sensor_ID: amsua_n18 + Sensor_ID: &Sensor_ID amsua_n18 EndianType: little_endian CoefficientPath: $(DATA)/crtm/ diff --git a/parm/atm/obs/config/amsua_n19.yaml b/parm/atm/obs/config/amsua_n19.yaml index a185d2485..f6f47c822 100644 --- a/parm/atm/obs/config/amsua_n19.yaml +++ b/parm/atm/obs/config/amsua_n19.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)amsua_n19.{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)amsua_n19.tm00.nc obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_amsua_n19_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_amsua_n19_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [brightnessTemperature] @@ -19,7 +19,7 @@ obs operator: Clouds: [Water, Ice] Cloud_Fraction: 1.0 obs options: - Sensor_ID: amsua_n19 + Sensor_ID: &Sensor_ID amsua_n19 EndianType: little_endian CoefficientPath: $(DATA)/crtm/ From f34c145c38b27343d4e0b796d1537995b6676c48 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 21 Nov 2023 11:14:05 -0600 Subject: [PATCH 31/58] Update diagnostic flags, add thinning and scan edge removal filters --- parm/atm/obs/config/amsua_metop-a.yaml | 73 ++++++++++++++--- parm/atm/obs/config/amsua_metop-b.yaml | 73 ++++++++++++++--- parm/atm/obs/config/amsua_metop-c.yaml | 72 ++++++++++++++--- parm/atm/obs/config/amsua_n15.yaml | 103 +++++++++++++----------- parm/atm/obs/config/amsua_n18.yaml | 104 ++++++++++++++----------- parm/atm/obs/config/amsua_n19.yaml | 103 +++++++++++++----------- 6 files changed, 363 insertions(+), 165 deletions(-) diff --git a/parm/atm/obs/config/amsua_metop-a.yaml b/parm/atm/obs/config/amsua_metop-a.yaml index 094142c14..34bceae43 100644 --- a/parm/atm/obs/config/amsua_metop-a.yaml +++ b/parm/atm/obs/config/amsua_metop-a.yaml @@ -54,8 +54,37 @@ obs bias: ratio for small dataset: 2.0 output file: $(DATA)/bc/$(APREFIX)amsua_metop-a.satbias_cov.nc4 -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + +obs post filters: +# Step 0-B: Create Diagnostic Flags # Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: @@ -225,7 +254,7 @@ obs post filters: channels: *amsua_metop-a_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 0-C: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -236,7 +265,32 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_metop-a_channels -# Step 2: CLW Retrieval Check (observation_based) +# Step 1: Remove observations from the edge of the scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + round_horizontal_bin_count_to_nearest: true + partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -246,10 +300,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 3: CLW Retrieval Check (background_based) +# Step 3B: CLW Retrieval Check (background_based) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,7 +313,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 4: Window channel sanity check @@ -295,7 +349,7 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject @@ -456,7 +510,7 @@ obs post filters: channels: *amsua_metop-a_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject @@ -478,7 +532,6 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject - diff --git a/parm/atm/obs/config/amsua_metop-b.yaml b/parm/atm/obs/config/amsua_metop-b.yaml index b6c0509d2..82d5fb950 100644 --- a/parm/atm/obs/config/amsua_metop-b.yaml +++ b/parm/atm/obs/config/amsua_metop-b.yaml @@ -54,8 +54,37 @@ obs bias: ratio for small dataset: 2.0 output file: $(DATA)/bc/$(APREFIX)amsua_metop-b.satbias_cov.nc4 -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + +obs post filters: +# Step 0-B: Create Diagnostic Flags # Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: @@ -225,7 +254,7 @@ obs post filters: channels: *amsua_metop-b_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 0-C: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -236,7 +265,32 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_metop-b_channels -# Step 2: CLW Retrieval Check (observation_based) +# Step 1: Remove observations from the edge of the scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + round_horizontal_bin_count_to_nearest: true + partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -246,10 +300,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 3: CLW Retrieval Check (background_based) +# Step 3B: CLW Retrieval Check (background_based) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,7 +313,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 4: Window channel sanity check @@ -295,7 +349,7 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject @@ -456,7 +510,7 @@ obs post filters: channels: *amsua_metop-b_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject @@ -478,7 +532,6 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject - diff --git a/parm/atm/obs/config/amsua_metop-c.yaml b/parm/atm/obs/config/amsua_metop-c.yaml index 1c4e1acca..818fd6d73 100644 --- a/parm/atm/obs/config/amsua_metop-c.yaml +++ b/parm/atm/obs/config/amsua_metop-c.yaml @@ -54,8 +54,37 @@ obs bias: ratio for small dataset: 2.0 output file: $(DATA)/bc/$(APREFIX)amsua_metop-c.satbias_cov.nc4 +obs pre filters: +# Step 0-A: Create Diagnostic Flags +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + obs post filters: -# Step 0-A: Create Diagnostic Flags +# Step 0-B: Create Diagnostic Flags # Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: @@ -225,7 +254,7 @@ obs post filters: channels: *amsua_metop-c_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 0-C: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -236,7 +265,32 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_metop-c_channels -# Step 2: CLW Retrieval Check (observation_based) +# Step 1: Remove observations from the edge of the scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + round_horizontal_bin_count_to_nearest: true + partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -246,10 +300,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 3: CLW Retrieval Check (background_based) +# Step 3B: CLW Retrieval Check (background_based) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,7 +313,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 4: Window channel sanity check @@ -295,7 +349,7 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject @@ -456,7 +510,7 @@ obs post filters: channels: *amsua_metop-c_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject @@ -478,6 +532,6 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject diff --git a/parm/atm/obs/config/amsua_n15.yaml b/parm/atm/obs/config/amsua_n15.yaml index 324f1ed96..511d7860a 100644 --- a/parm/atm/obs/config/amsua_n15.yaml +++ b/parm/atm/obs/config/amsua_n15.yaml @@ -54,48 +54,36 @@ obs bias: ratio for small dataset: 2.0 output file: $(DATA)/bc/$(APREFIX)amsua_n15.satbias_cov.nc4 -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n15_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n15_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n15_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature + filter variables: + - name: brightnessTemperature channels: *amsua_n15_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false +obs post filters: # Step 0-B: Calculate derived variables # Calculate CLW retrieved from observation - filter: Variable Assignment @@ -225,7 +213,7 @@ obs post filters: channels: *amsua_n15_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 0-C: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -236,7 +224,32 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_n15_channels -# Step 2: CLW Retrieval Check (observation_based) +# Step 1: Remove observations from the edge of the scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + round_horizontal_bin_count_to_nearest: true + partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -246,10 +259,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 3: CLW Retrieval Check (background_based) +# Step 3B: CLW Retrieval Check (background_based) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,7 +272,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 4: Window channel sanity check @@ -295,7 +308,7 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject @@ -456,7 +469,7 @@ obs post filters: channels: *amsua_n15_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject @@ -478,6 +491,6 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject diff --git a/parm/atm/obs/config/amsua_n18.yaml b/parm/atm/obs/config/amsua_n18.yaml index 51648d34a..7d749c4bc 100644 --- a/parm/atm/obs/config/amsua_n18.yaml +++ b/parm/atm/obs/config/amsua_n18.yaml @@ -54,48 +54,36 @@ obs bias: ratio for small dataset: 2.0 output file: $(DATA)/bc/$(APREFIX)amsua_n18.satbias_cov.nc4 -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n18_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n18_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n18_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature + filter variables: + - name: brightnessTemperature channels: *amsua_n18_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false +obs post filters: # Step 0-B: Calculate derived variables # Calculate CLW retrieved from observation - filter: Variable Assignment @@ -225,7 +213,7 @@ obs post filters: channels: *amsua_n18_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 0-C: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -236,7 +224,32 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_n18_channels -# Step 2: CLW Retrieval Check (observation_based) +# Step 1: Remove observations from the edge of the scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + round_horizontal_bin_count_to_nearest: true + partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -246,10 +259,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 3: CLW Retrieval Check (background_based) +# Step 3B: CLW Retrieval Check (background_based) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,7 +272,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 4: Window channel sanity check @@ -295,7 +308,7 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject @@ -456,7 +469,7 @@ obs post filters: channels: *amsua_n18_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject @@ -478,7 +491,6 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject - diff --git a/parm/atm/obs/config/amsua_n19.yaml b/parm/atm/obs/config/amsua_n19.yaml index f6f47c822..a940447d8 100644 --- a/parm/atm/obs/config/amsua_n19.yaml +++ b/parm/atm/obs/config/amsua_n19.yaml @@ -54,48 +54,36 @@ obs bias: ratio for small dataset: 2.0 output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias_cov.nc4 -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature + filter variables: + - name: brightnessTemperature channels: *amsua_n19_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false +obs post filters: # Step 0-B: Calculate derived variables # Calculate CLW retrieved from observation - filter: Variable Assignment @@ -225,7 +213,7 @@ obs post filters: channels: *amsua_n19_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 0-C: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -236,7 +224,32 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_n19_channels -# Step 2: CLW Retrieval Check (observation_based) +# Step 1: Remove observations from the edge of the scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + round_horizontal_bin_count_to_nearest: true + partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -246,10 +259,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 3: CLW Retrieval Check (background_based) +# Step 3B: CLW Retrieval Check (background_based) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,7 +272,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 4: Window channel sanity check @@ -295,7 +308,7 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject @@ -456,7 +469,7 @@ obs post filters: channels: *amsua_n19_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject @@ -478,6 +491,6 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject From d7a03787b79860c90bfe6e3a1643a6e3d1dafb4a Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 21 Nov 2023 11:15:02 -0600 Subject: [PATCH 32/58] Remove extra comments --- parm/atm/obs/config/atms_n20.yaml | 1 - parm/atm/obs/config/atms_npp.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/parm/atm/obs/config/atms_n20.yaml b/parm/atm/obs/config/atms_n20.yaml index 93590ab7a..9a85b5616 100644 --- a/parm/atm/obs/config/atms_n20.yaml +++ b/parm/atm/obs/config/atms_n20.yaml @@ -56,7 +56,6 @@ obs bias: obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature diff --git a/parm/atm/obs/config/atms_npp.yaml b/parm/atm/obs/config/atms_npp.yaml index bb668ab9d..8fab19c05 100644 --- a/parm/atm/obs/config/atms_npp.yaml +++ b/parm/atm/obs/config/atms_npp.yaml @@ -56,7 +56,6 @@ obs bias: obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature From 2dd725fedfcc6290e32ea501a358a0d022ced293 Mon Sep 17 00:00:00 2001 From: BrettHoover-NOAA <98188219+BrettHoover-NOAA@users.noreply.github.com> Date: Tue, 21 Nov 2023 11:55:35 -0600 Subject: [PATCH 33/58] [End-to-End Test Code Sprint] Add SEVIRI METEOSAT-8 satwinds to end-to-end testing (#764) Adding satwinds from the Spinning Enhanced Visible and InfraRed Imager (SEVIRI) from METEOSAT-8 to GDASApp end-to-end testing new files include: parm/atm/obs/config/satwind_seviri_m8.yaml: QC filter YAML for SEVIRI METEOSAT-8 satwinds parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json: JSON containing data format, sensor, and satellite information for SEVIRI METEOSAT-8 satwinds ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.py: bufr2ioda code for extracting SEVIRI METEOSAT-8 satwinds from BUFR End-to-end testing results are provided in #758 - Note: Thinning filter is provided in YAML but turned off due to irreconcilable differences with GSI in testing. --------- Co-authored-by: Brett Hoover --- parm/atm/obs/config/satwind_seviri_m8.yaml | 405 ++++++++++++++++ .../bufr2ioda_satwind_amv_seviri.json | 15 + .../bufr2ioda/bufr2ioda_satwind_amv_seviri.py | 441 ++++++++++++++++++ 3 files changed, 861 insertions(+) create mode 100644 parm/atm/obs/config/satwind_seviri_m8.yaml create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.py diff --git a/parm/atm/obs/config/satwind_seviri_m8.yaml b/parm/atm/obs/config/satwind_seviri_m8.yaml new file mode 100644 index 000000000..70ada0f9d --- /dev/null +++ b/parm/atm/obs/config/satwind_seviri_m8.yaml @@ -0,0 +1,405 @@ +obs space: + name: satwind_seviri_m8 + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)satwnd.seviri_m8.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_satwind_seviri_m8_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + simulated variables: [windEastward, windNorthward] + +obs operator: + name: VertInterp + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +linear obs operator: + name: VertInterp + +# NOTE: Tests using the Gaussian Thinning filter (below) to duplicate GSI's thinning of SEVIRI/METEOSAT-8 satwinds +# results in more JEDI satwinds in the diag file than in GSI, but far fewer JEDI satwinds assimilated than +# GSI. JEDI under-counts assimilated winds by roughly 25-40%, relative to GSI, and this under-count is not +# even including the temporal thinning which is applied in GSI but not JEDI (by this filter below). See +# GDASApp Issue #758 for details: https://github.com/NOAA-EMC/GDASApp/issues/758 +#obs pre filters: +#- filter: Gaussian Thinning +# where: +# - variable: ObsType/windEastward +# is_in: 243, 253 +# horizontal_mesh: 200 +# vertical_mesh: 10000 +# use_reduced_horizontal_grid: true +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + +obs prior filters: +# Apply variable changes needed for wind scaling +# For wind observations with pressure provided +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +obs post filters: +# Assign the initial observation error, based on height/pressure +# Hard-wiring to prepobs_errtable.global by Type +# ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML + +# Type 243 (MVIRI/SEVIRI VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 243 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 253 (MVIRI/SEVERI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 253 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 254 (MVIRI/SEVIRI WV, both cloud-top and clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 254 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.5,6.1,6.,6.5,7.3,7.6,7.,7.5,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + + +# sanity-check criteria +# Observation Range Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -130. + maxvalue: 130. + action: + name: reject + +# Velocity Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Velocity + maxvalue: 130. + action: + name: reject + +# GSI read routine QC (part-1) +# Exclude Type 254 with windComputationMethod==5 (clear-sky WV) --- obs tossed without passing to setup routine +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_in: 254 + - variable: MetaData/windComputationMethod + is_in: 5 + action: + name: reject + +# Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/satelliteZenithAngle + maxvalue: 68. + action: + name: reject + + +# Exclude data over non-water surface type where latitude > 20N for Type 253 (IRLW) --- obs tossed and not passed to setup routine +# Notes: This check was missing, so added (eliu) +# Replace land_type_index_NPOSS with water_area_fraction (eliu) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 253 + - variable: MetaData/latitude + minvalue: 20. + test variables: + - name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + +# GSI read routine QC (part-2) +# Reject obs with qualityInformationWithoutForecast < 85. OR > 100. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 85. + maxvalue: 100. + action: + name: reject + +# GSI setupw routine QC +# Reject any ob Type [240–260] when pressure greater than 950 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240-260 + test variables: + - name: MetaData/pressure + maxvalue: 95001. + action: + name: reject + +# EUMETSAT IR (253) reject when pressure between 401 and 801 mb. +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 40101. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 253 + action: + name: reject + +# EUMET VIS (243) reject when pressure less than 700 mb. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 243 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# EUMET WV (254) reject when pressure greater than 399 mb. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 254 + test variables: + - name: MetaData/pressure + maxvalue: 39900. + action: + name: reject + +# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +# Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation +# Turn this check off for now. +# Need to check if troposphere pressure was implemented correctly in fv3-jed +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/tropopause_pressure + value: MetaData/pressure + minvalue: -5000. # 50 hPa above tropopause level, negative p-diff + action: + name: reject + +# All satwinds must adjust errors based on ObsErrorFactorPressureCheck +# prior to the SPDB check (i.e. the gross-error check). The gross-error +# check uses the adjusted errors for error-bound tightening and rejection, +# so this check has to come first. This check will inflate errors for obs +# that are too close to either the model top or bottom. +# Notes (eliu): GMAO added a required parameter: adjusted_error_name. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + surface_obs: false + variable: windEastward + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All satwinds subject to a gross-error check that contains significant +# modifiers for satwinds with a negative speed-bias. ALL wind gross-error +# checks are currently being done by the SatWindsSPDBCheck. +# CLEARED +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +# Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). +# : Turn off duplicate check for now. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# We are extending this to an additional filter that inflates final ob-errors across-the-board by +# 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), +# you would want to remove this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +# End of Filters diff --git a/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json b/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json new file mode 100644 index 000000000..fd90d4a02 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json @@ -0,0 +1,15 @@ +{ + "data_format" : "bufr_d", + "data_type" : "satwnd", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : [ "NC005067", "NC005068", "NC005069" ], + "data_description" : "EUMETSAT SATWIND, SEVIRI IR(LW); EUMETSAT SATWIND, SEVIRI WV-IMG/DL; EUMETSAT SATWIND, SEVIRI VIS", + "data_provider" : "EUMETSAT", + "sensor_info" : { "sensor_name": "SEVIRI", "sensor_full_name": "Spinning Enhanced Visible and InfraRed Imager", "sensor_id": 999 }, + "satellite_info" : [ + { "satellite_name": "M8", "satellite_full_name": "METEOSAT-8", "satellite_id": 55, "launch time": "YYYYMMDD" }, + ] +} diff --git a/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.py b/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.py new file mode 100755 index 000000000..c28274948 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python3 +import argparse +import numpy as np +import numpy.ma as ma +from pyiodaconv import bufr +import calendar +import json +import time +import math +import datetime +import os +from datetime import datetime +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger + +# ==================================================================== +# Satellite Winds (AMV) BUFR dump file for SEVIRI/METEOSAT +# ==================================================================== +# All subsets contain all spectral bands: NC005067 NC005068 NC005069 +# ==================================================================== +# Spectral Band | Code (002023) | ObsType +# -------------------------------------------------------------------- +# IRLW (Freq < 5E+13) | Method 1 | 253 +# VIS | Method 2 | 243 +# WV Cloud Top | Method 3 | 254 +# WV Clear Sky/ Deep Layer | Method 5 | 254 +# ==================================================================== + +# Define and initialize global variables +global float32_fill_value +global int32_fill_value +global int64_fill_value + +float32_fill_value = np.float32(0) +int32_fill_value = np.int32(0) +int64_fill_value = np.int64(0) + + +def Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd): + + uob = (-wspd * np.sin(np.radians(wdir))).astype(np.float32) + vob = (-wspd * np.cos(np.radians(wdir))).astype(np.float32) + + return uob, vob + + +def Get_ObsType(swcm, chanfreq): + + obstype = swcm.copy() + + # Use numpy vectorized operations + obstype = np.where(swcm == 5, 254, obstype) # WVCA/DL + obstype = np.where(swcm == 3, 254, obstype) # WVCT + obstype = np.where(swcm == 2, 243, obstype) # VIS + obstype = np.where(swcm == 1, 253, obstype) # IRLW + + if not np.any(np.isin(obstype, [243, 253, 254])): + raise ValueError("Error: Unassigned ObsType found ... ") + + return obstype + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + subsets = config["subsets"] + data_format = config["data_format"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + satellite_info_array = config["satellite_info"] + sensor_name = config["sensor_info"]["sensor_name"] + sensor_full_name = config["sensor_info"]["sensor_full_name"] + sensor_id = config["sensor_info"]["sensor_id"] + + # Get derived parameters + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + reference_time = datetime.strptime(cycle, "%Y%m%d%H") + reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") + + # General informaton + converter = 'BUFR to IODA Converter' + process_level = 'Level-2' + platform_description = 'METEOSAT-8' + sensor_description = 'Spinning Enhanced Visible and InfraRed Imager' + + logger.info(f'sensor_name = {sensor_name}') + logger.info(f'sensor_full_name = {sensor_full_name}') + logger.info(f'sensor_id = {sensor_id}') + logger.info(f'reference_time = {reference_time}') + + bufrfile = f"{cycle_type}.t{hh}z.{data_type}.tm00.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), 'atmos', bufrfile) + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + logger.info('Making QuerySet') + q = bufr.QuerySet(subsets) + + # MetaData + q.add('latitude', '*/CLATH') + q.add('longitude', '*/CLONH') + q.add('satelliteId', '*/SAID') + q.add('year', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('second', '*/SECO') + q.add('satelliteZenithAngle', '*/SAZA') + q.add('sensorCentralFrequency', '*/SCCF') + q.add('pressure', '*/PRLC[1]') + + # Processing Center + q.add('dataProviderOrigin', '*/OGCE') + +# # Quality Infomation (Quality Indicator w/o forecast) + q.add('qualityInformationWithoutForecast', '*/AMVQIC{2}/PCCF') + + # Wind Retrieval Method Information + q.add('windComputationMethod', '*/SWCM') + + # ObsValue + q.add('windDirection', '*/WDIR') + q.add('windSpeed', '*/WSPD') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f'Processing time for making QuerySet : {running_time} seconds') + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + logger.info('Executing QuerySet to get ResultSet') + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + # MetaData + satid = r.get('satelliteId') + year = r.get('year') + month = r.get('month') + day = r.get('day') + hour = r.get('hour') + minute = r.get('minute') + second = r.get('second') + lat = r.get('latitude') + lon = r.get('longitude') + satzenang = r.get('satelliteZenithAngle') + pressure = r.get('pressure', type='float') + chanfreq = r.get('sensorCentralFrequency', type='float') + + # Processing Center + ogce = r.get('dataProviderOrigin') + + # Quality Information + qifn = r.get('qualityInformationWithoutForecast', type='float') + + # Wind Retrieval Method Information + swcm = r.get('windComputationMethod') + + # ObsValue + # Wind direction and Speed + wdir = r.get('windDirection', type='float') + wspd = r.get('windSpeed') + + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + timestamp = r.get_datetime('year', 'month', 'day', 'hour', 'minute', 'second').astype(np.int64) + + # Check BUFR variable generic dimension and type + + # Global variables declaration + # Set global fill values + float32_fill_value = satzenang.fill_value + int32_fill_value = satid.fill_value + int64_fill_value = timestamp.fill_value.astype(np.int64) + + end_time = time.time() + running_time = end_time - start_time + logger.info(f'Processing time for executing QuerySet to get ResultSet : {running_time} seconds') + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + logger.info('Creating derived variables') + logger.debug('Creating derived variables - wind components (uob and vob)') + + uob, vob = Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd) + + logger.debug(f' uob min/max = {uob.min()} {uob.max()}') + logger.debug(f' vob min/max = {vob.min()} {vob.max()}') + + obstype = Get_ObsType(swcm, chanfreq) + + height = np.full_like(pressure, fill_value=pressure.fill_value, dtype=np.float32) + stnelev = np.full_like(pressure, fill_value=pressure.fill_value, dtype=np.float32) + + end_time = time.time() + running_time = end_time - start_time + logger.info(f'Processing time for creating derived variables : {running_time} seconds') + + # ===================================== + # Split output based on satellite id + # Create IODA ObsSpace + # Write IODA output + # ===================================== + logger.info('Create IODA ObsSpace and Write IODA output based on satellite ID') + + # Find unique satellite identifiers in data to process + unique_satids = np.unique(satid) + logger.info(f'Number of Unique satellite identifiers: {len(unique_satids)}') + logger.info(f'Unique satellite identifiers: {unique_satids}') + + logger.debug(f'Loop through unique satellite identifier {unique_satids}') + total_ob_processed = 0 + for sat in unique_satids.tolist(): + start_time = time.time() + + matched = False + for satellite_info in satellite_info_array: + if (satellite_info["satellite_id"] == sat): + matched = True + satellite_id = satellite_info["satellite_id"] + satellite_name = satellite_info["satellite_name"] + satinst = sensor_name.lower()+'_'+satellite_name.lower() + logger.debug(f'Split data for {satinst} satid = {sat}') + + if matched: + + # Define a boolean mask to subset data from the original data object + mask = satid == sat + # MetaData + lon2 = lon[mask] + lat2 = lat[mask] + timestamp2 = timestamp[mask] + satid2 = satid[mask] + satzenang2 = satzenang[mask] + chanfreq2 = chanfreq[mask] + obstype2 = obstype[mask] + pressure2 = pressure[mask] + height2 = height[mask] + stnelev2 = stnelev[mask] + + # Processing Center + ogce2 = ogce[mask] + + # QC Info + qifn2 = qifn[mask] + + # Method + swcm2 = swcm[mask] + + # ObsValue + wdir2 = wdir[mask] + wspd2 = wspd[mask] + uob2 = uob[mask] + vob2 = vob[mask] + + # Timestamp Range + timestamp2_min = datetime.fromtimestamp(timestamp2.min()) + timestamp2_max = datetime.fromtimestamp(timestamp2.max()) + + # Check unique observation time + unique_timestamp2 = np.unique(timestamp2) + logger.debug(f'Processing output for satid {sat}') + + # Create the dimensions + dims = { + 'Location': np.arange(0, wdir2.shape[0]) + } + + # Create IODA ObsSpace + iodafile = f"{cycle_type}.t{hh}z.{data_type}.{satinst}.tm00.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.info(f'Create output file : {OUTPUT_PATH}') + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug('Write global attributes') + obsspace.write_attr('Converter', converter) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeReference', reference_time) + obsspace.write_attr('datetimeRange', [str(timestamp2_min), str(timestamp2_max)]) + obsspace.write_attr('sensor', sensor_id) + obsspace.write_attr('platform', satellite_id) + obsspace.write_attr('platformCommonName', satellite_name) + obsspace.write_attr('sensorCommonName', sensor_name) + obsspace.write_attr('processingLevel', process_level) + obsspace.write_attr('platformLongDescription', platform_description) + obsspace.write_attr('sensorLongDescription', sensor_description) + + # Create IODA variables + logger.debug('Write variables: name, type, units, and attributes') + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon2.dtype, fillval=lon2.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon2) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat2.dtype, fillval=lat2.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat2) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=np.int64, fillval=int64_fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(timestamp2) + + # Satellite Identifier + obsspace.create_var('MetaData/satelliteIdentifier', dtype=satid2.dtype, fillval=satid2.fill_value) \ + .write_attr('long_name', 'Satellite Identifier') \ + .write_data(satid2) + + # Sensor Zenith Angle + obsspace.create_var('MetaData/satelliteZenithAngle', dtype=satzenang2.dtype, fillval=satzenang2.fill_value) \ + .write_attr('units', 'degree') \ + .write_attr('valid_range', np.array([0, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Satellite Zenith Angle') \ + .write_data(satzenang2) + + # Sensor Centrall Frequency + obsspace.create_var('MetaData/sensorCentralFrequency', dtype=chanfreq2.dtype, fillval=chanfreq2.fill_value) \ + .write_attr('units', 'Hz') \ + .write_attr('long_name', 'Satellite Channel Center Frequency') \ + .write_data(chanfreq2) + + # Data Provider + obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce2.dtype, fillval=ogce2.fill_value) \ + .write_attr('long_name', 'Identification of Originating/Generating Center') \ + .write_data(ogce2) + + # Quality: Percent Confidence - Quality Information Without Forecast + obsspace.create_var('MetaData/qualityInformationWithoutForecast', dtype=qifn2.dtype, fillval=qifn2.fill_value) \ + .write_attr('long_name', 'Quality Information Without Forecast') \ + .write_data(qifn2) + + # Wind Computation Method + obsspace.create_var('MetaData/windComputationMethod', dtype=swcm2.dtype, fillval=swcm2.fill_value) \ + .write_attr('long_name', 'Satellite-derived Wind Computation Method') \ + .write_data(swcm2) + + # ObsType based on computation method/spectral band + obsspace.create_var('ObsType/windEastward', dtype=obstype2.dtype, fillval=swcm2.fill_value) \ + .write_attr('long_name', 'Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') \ + .write_data(obstype2) + + # ObsType based on computation method/spectral band + obsspace.create_var('ObsType/windNorthward', dtype=obstype2.dtype, fillval=swcm2.fill_value) \ + .write_attr('long_name', 'Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') \ + .write_data(obstype2) + + # Pressure + obsspace.create_var('MetaData/pressure', dtype=pressure2.dtype, fillval=pressure2.fill_value) \ + .write_attr('units', 'pa') \ + .write_attr('long_name', 'Pressure') \ + .write_data(pressure2) + + # Height (mimic prepbufr) + obsspace.create_var('MetaData/height', dtype=height2.dtype, fillval=height2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Height of Observation') \ + .write_data(height2) + + # Station Elevation (mimic prepbufr) + obsspace.create_var('MetaData/stationElevation', dtype=stnelev2.dtype, fillval=stnelev2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Station Elevation') \ + .write_data(stnelev2) + + # U-Wind Component + obsspace.create_var('ObsValue/windEastward', dtype=uob2.dtype, fillval=wspd2.fill_value) \ + .write_attr('units', 'm s-1') \ + .write_attr('long_name', 'Eastward Wind Component') \ + .write_data(uob2) + + # V-Wind Component + obsspace.create_var('ObsValue/windNorthward', dtype=vob2.dtype, fillval=wspd2.fill_value) \ + .write_attr('units', 'm s-1') \ + .write_attr('long_name', 'Northward Wind Component') \ + .write_data(vob2) + + end_time = time.time() + running_time = end_time - start_time + total_ob_processed += len(satid2) + logger.debug(f'Number of observation processed : {len(satid2)}') + logger.debug(f'Processing time for splitting and output IODA for {satinst} : {running_time} seconds') + + else: + logger.info(f"Do not find this satellite id in the configuration: satid = {sat}") + + logger.info("All Done!") + logger.info(f'Total number of observation processed : {total_ob_processed}') + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('BUFR2IODA_satwind_amv_ahi.py', level=log_level, colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.info(f"Total running time: {running_time} seconds") From 25ab8ce43850f16d864234b40a7e085ff7ffb991 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 21 Nov 2023 16:29:58 -0600 Subject: [PATCH 34/58] Do nothing. --- CMakeLists.txt | 3 ++- parm/atm/variational/3dvar_dripcg.yaml | 36 +------------------------ parm/ioda/bufr2ioda/bufr2ioda_atms.yaml | 3 --- 3 files changed, 3 insertions(+), 39 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b26b64b5b..60ea4135c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,7 +80,8 @@ if(BUILD_GDASBUNDLE) option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" BRANCH develop ) option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop ) +# ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop ) + ecbuild_bundle( PROJECT ufo GIT "https://github.com/JCSDA-internal/ufo.git" BRANCH feature/mwrad ) # FMS and FV3 dynamical core ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml index abd945842..2f9ccb76f 100644 --- a/parm/atm/variational/3dvar_dripcg.yaml +++ b/parm/atm/variational/3dvar_dripcg.yaml @@ -33,7 +33,7 @@ variational: minimizer: algorithm: DRIPCG iterations: - - ninner: 2 + - ninner: 1 gradient norm reduction: 1e-10 test: on geometry: @@ -50,40 +50,6 @@ variational: field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml diagnostics: departures: bkgmob - - ninner: 4 - gradient norm reduction: 1e-10 - test: on - geometry: - fms initialization: - namelist filename: ./fv3jedi/fmsmpp.nml - field table filename: ./fv3jedi/field_table - akbk: ./fv3jedi/akbk.nc4 - layout: - - $(layout_x) - - $(layout_y) - npx: $(npx_anl) - npy: $(npy_anl) - npz: $(npz_anl) - field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml - diagnostics: - departures: bkgmob1 final: diagnostics: departures: anlmob - increment: - output: - filetype: auxgrid - gridtype: gaussian - filename: ./anl/atminc. - geometry: - fms initialization: - namelist filename: ./fv3jedi/fmsmpp.nml - field table filename: ./fv3jedi/field_table - akbk: ./fv3jedi/akbk.nc4 - layout: - - $(layout_x) - - $(layout_y) - npx: $(npx_anl) - npy: $(npy_anl) - npz: $(npz_anl) - field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml diff --git a/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml b/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml index 0f1d69fb7..532e8510c 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml +++ b/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml @@ -27,11 +27,8 @@ observations: satelliteInstrument: query: "*/SIID" - # The type should be integer. - # However, UFO/CRTM requred this variable to be float fieldOfViewNumber: query: "*/FOVN" -# type: float heightOfStation: query: "*/HMSL" From 94abd9aaa6b1960018eabc427212208fcb7a46bc Mon Sep 17 00:00:00 2001 From: BrettHoover-NOAA <98188219+BrettHoover-NOAA@users.noreply.github.com> Date: Mon, 27 Nov 2023 07:50:27 -0600 Subject: [PATCH 35/58] Added SEVIRI/METEOSAT-11 satwinds to end-to-end validation (#767) Small code-change to include SEVIRI/METEOSAT-11 satwinds in end-to-end validation. This uses the hook-ins already provided by the SEVIRI/METEOSAT-8 satwinds code that was merged in https://github.com/NOAA-EMC/GDASApp/pull/764. New file: parm/atm/obs/config/satwind_seviri_m11.yaml - this is nearly a direct copy of parm/atm/obs/config/satwind_seviri_m8.yaml, since they utilize the same filters. Only the header information has changed. Modified file: parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json - METEOSAT-11 was added to the `satellite_info` group End-to-end testing looks very similar to what was seen for SEVIRI/METEOSAT-8 satwinds, detailed in https://github.com/NOAA-EMC/GDASApp/issues/758. A brief run-down: (LW)IR winds are in QC agreement down to a 0.02% difference (cloud-top)WV winds are in QC agreement down to a 0.20% difference VIS winds differ by 4.5%, but there are only 22 accepted winds in GSI and 23 in JEDI, the difference is a single satwind All ob, HofX, and ob-minus-HofX differences look good Increment summary, from gdasatmanlrun.log: ``` 0: ---------------------------------------------------------------------------------------------------- 0: Increment print | number of fields = 8 | cube sphere face size: C768 0: eastward_wind | Min:-3.479765e-01 Max:+3.430433e-01 RMS:+4.990993e-04 0: northward_wind | Min:-4.226316e-01 Max:+3.376561e-01 RMS:+4.889429e-04 0: air_temperature | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: surface_pressure | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: specific_humidity | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: cloud_liquid_ice | Min:+0.000000e+00 Max:+1.618770e-20 RMS:+1.293217e-23 0: cloud_liquid_water | Min:+0.000000e+00 Max:+1.474788e-19 RMS:+2.167418e-22 0: ozone_mass_mixing_ratio | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: ---------------------------------------------------------------------------------------------------- ``` Co-authored-by: Brett Hoover --- parm/atm/obs/config/satwind_seviri_m11.yaml | 405 ++++++++++++++++++ .../bufr2ioda_satwind_amv_seviri.json | 1 + 2 files changed, 406 insertions(+) create mode 100644 parm/atm/obs/config/satwind_seviri_m11.yaml diff --git a/parm/atm/obs/config/satwind_seviri_m11.yaml b/parm/atm/obs/config/satwind_seviri_m11.yaml new file mode 100644 index 000000000..763e59fda --- /dev/null +++ b/parm/atm/obs/config/satwind_seviri_m11.yaml @@ -0,0 +1,405 @@ +obs space: + name: satwind_seviri_m11 + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)satwnd.seviri_m11.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_satwind_seviri_m11_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + simulated variables: [windEastward, windNorthward] + +obs operator: + name: VertInterp + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +linear obs operator: + name: VertInterp + +# NOTE: Tests using the Gaussian Thinning filter (below) to duplicate GSI's thinning of SEVIRI/METEOSAT-8 satwinds +# results in more JEDI satwinds in the diag file than in GSI, but far fewer JEDI satwinds assimilated than +# GSI. JEDI under-counts assimilated winds by roughly 25-40%, relative to GSI, and this under-count is not +# even including the temporal thinning which is applied in GSI but not JEDI (by this filter below). See +# GDASApp Issue #758 for details: https://github.com/NOAA-EMC/GDASApp/issues/758 +#obs pre filters: +#- filter: Gaussian Thinning +# where: +# - variable: ObsType/windEastward +# is_in: 243, 253 +# horizontal_mesh: 200 +# vertical_mesh: 10000 +# use_reduced_horizontal_grid: true +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + +obs prior filters: +# Apply variable changes needed for wind scaling +# For wind observations with pressure provided +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +obs post filters: +# Assign the initial observation error, based on height/pressure +# Hard-wiring to prepobs_errtable.global by Type +# ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML + +# Type 243 (MVIRI/SEVIRI VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 243 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 253 (MVIRI/SEVERI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 253 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 254 (MVIRI/SEVIRI WV, both cloud-top and clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 254 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.5,6.1,6.,6.5,7.3,7.6,7.,7.5,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + + +# sanity-check criteria +# Observation Range Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -130. + maxvalue: 130. + action: + name: reject + +# Velocity Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Velocity + maxvalue: 130. + action: + name: reject + +# GSI read routine QC (part-1) +# Exclude Type 254 with windComputationMethod==5 (clear-sky WV) --- obs tossed without passing to setup routine +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_in: 254 + - variable: MetaData/windComputationMethod + is_in: 5 + action: + name: reject + +# Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/satelliteZenithAngle + maxvalue: 68. + action: + name: reject + + +# Exclude data over non-water surface type where latitude > 20N for Type 253 (IRLW) --- obs tossed and not passed to setup routine +# Notes: This check was missing, so added (eliu) +# Replace land_type_index_NPOSS with water_area_fraction (eliu) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 253 + - variable: MetaData/latitude + minvalue: 20. + test variables: + - name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + +# GSI read routine QC (part-2) +# Reject obs with qualityInformationWithoutForecast < 85. OR > 100. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 85. + maxvalue: 100. + action: + name: reject + +# GSI setupw routine QC +# Reject any ob Type [240–260] when pressure greater than 950 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240-260 + test variables: + - name: MetaData/pressure + maxvalue: 95001. + action: + name: reject + +# EUMETSAT IR (253) reject when pressure between 401 and 801 mb. +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 40101. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 253 + action: + name: reject + +# EUMET VIS (243) reject when pressure less than 700 mb. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 243 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# EUMET WV (254) reject when pressure greater than 399 mb. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 254 + test variables: + - name: MetaData/pressure + maxvalue: 39900. + action: + name: reject + +# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +# Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation +# Turn this check off for now. +# Need to check if troposphere pressure was implemented correctly in fv3-jed +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/tropopause_pressure + value: MetaData/pressure + minvalue: -5000. # 50 hPa above tropopause level, negative p-diff + action: + name: reject + +# All satwinds must adjust errors based on ObsErrorFactorPressureCheck +# prior to the SPDB check (i.e. the gross-error check). The gross-error +# check uses the adjusted errors for error-bound tightening and rejection, +# so this check has to come first. This check will inflate errors for obs +# that are too close to either the model top or bottom. +# Notes (eliu): GMAO added a required parameter: adjusted_error_name. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + surface_obs: false + variable: windEastward + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All satwinds subject to a gross-error check that contains significant +# modifiers for satwinds with a negative speed-bias. ALL wind gross-error +# checks are currently being done by the SatWindsSPDBCheck. +# CLEARED +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +# Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). +# : Turn off duplicate check for now. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# We are extending this to an additional filter that inflates final ob-errors across-the-board by +# 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), +# you would want to remove this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +# End of Filters diff --git a/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json b/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json index fd90d4a02..3fad3c258 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json +++ b/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json @@ -11,5 +11,6 @@ "sensor_info" : { "sensor_name": "SEVIRI", "sensor_full_name": "Spinning Enhanced Visible and InfraRed Imager", "sensor_id": 999 }, "satellite_info" : [ { "satellite_name": "M8", "satellite_full_name": "METEOSAT-8", "satellite_id": 55, "launch time": "YYYYMMDD" }, + { "satellite_name": "M11", "satellite_full_name": "METEOSAT-11", "satellite_id": 70, "launch time": "YYYYMMDD" }, ] } From e7fc7cb3e4c9a574fa549fc3fed28beda0544f01 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:31:56 -0500 Subject: [PATCH 36/58] A few updates for ATMS end-to-end testing (#768) In [ previous PR](https://github.com/NOAA-EMC/GDASApp/pull/757) for ATMS, the end-to-end testing was done without data thinning. And the excessive QC filtering in clear-sky areas, compared to GSI, was found. (See [plots](https://github.com/NOAA-EMC/GDASApp/pull/757#issuecomment-1819753244)) This PR includes two fixes for ATMS in the end-to-end testing and one new feature: 1. fix the excess QC filtering in clear-sky areas. ---> This has a paring [UFO PR #3122](https://github.com/JCSDA-internal/ufo/pull/3122) 2. add data thinning 3. add the diagnostic flags (QC) --- **This can [reproduce QC flags](https://github.com/NOAA-EMC/GDASApp/pull/768#issuecomment-1828209921) from GSI in UFO** **To test updates in this PR, please check out the UFO branch: [feature/satrad](https://github.com/JCSDA-internal/ufo/tree/feature/satrad) from JCSDA-internal in gdas-validation. This branch consolidates all proposed code changes to UFO for gdas validation: UFO PR #https://github.com/JCSDA-internal/ufo/pull/3122 UFO PR #https://github.com/JCSDA-internal/ufo/pull/3121 UFO PR #https://github.com/JCSDA-internal/ufo/pull/3094 --------- Co-authored-by: Cory Martin --- parm/atm/obs/config/atms_n20.yaml | 63 ++++++++++---- parm/atm/obs/config/atms_npp.yaml | 63 ++++++++++---- parm/atm/obs/testing/atms_n20.yaml | 111 ++++++++++++++---------- parm/atm/obs/testing/atms_n20_noqc.yaml | 21 +++-- parm/atm/obs/testing/atms_npp.yaml | 110 +++++++++++++---------- parm/atm/obs/testing/atms_npp_noqc.yaml | 21 +++-- 6 files changed, 242 insertions(+), 147 deletions(-) diff --git a/parm/atm/obs/config/atms_n20.yaml b/parm/atm/obs/config/atms_n20.yaml index 93590ab7a..2a29f5d95 100644 --- a/parm/atm/obs/config/atms_n20.yaml +++ b/parm/atm/obs/config/atms_n20.yaml @@ -18,6 +18,7 @@ obs operator: Absorbers: [H2O,O3] Clouds: [Water, Ice] Cloud_Fraction: 1.0 + Cloud_Seeding: true obs options: Sensor_ID: &Sensor_ID atms_n20 EndianType: little_endian @@ -56,7 +57,6 @@ obs bias: obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature @@ -83,6 +83,9 @@ obs pre filters: - name: InterChannelConsistency initial value: false force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false obs post filters: # Step 0-B: Calculate derived variables @@ -224,7 +227,7 @@ obs post filters: channels: *atms_n20_channels coefs: [1, -1] -# Step 0-C: Assign initial all-sky observation error +# Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -235,7 +238,7 @@ obs post filters: name: InitialObsError@DerivedMetaData channels: *atms_n20_channels -# Step 1: Remove observations from the edge of the scan +# Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: - name: brightnessTemperature @@ -250,15 +253,16 @@ obs post filters: - name: reject # Step 2: Data Thinning -#- filter: Gaussian Thinning -# horizontal_mesh: 145 -# use_reduced_horizontal_grid: true -## round_horizontal_bin_count_to_nearest: true -## partition_longitude_bins_using_mesh: true -# actions: -# - name: set -# flag: Thinning -# - name: reject +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject # Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -286,7 +290,7 @@ obs post filters: flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -338,7 +342,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: ObsErrorFactorTopo@DerivedMetaData @@ -361,7 +365,7 @@ obs post filters: name: ObsErrorFactorTopo@DerivedMetaData channels: *atms_n20_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: ObsErrorFactorTransmitTop@DerivedMetaData @@ -383,7 +387,7 @@ obs post filters: name: ObsErrorFactorTransmitTop@DerivedMetaData channels: *atms_n20_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: ObsErrorFactorSurfJacobian@DerivedMetaData @@ -409,7 +413,7 @@ obs post filters: name: ObsErrorFactorSurfJacobian@DerivedMetaData channels: *atms_n20_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: ObsErrorFactorSituDepend@DerivedMetaData @@ -504,7 +508,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -518,7 +522,7 @@ obs post filters: sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1] maxvalue: 1.0e-12 @@ -528,3 +532,24 @@ obs post filters: ignore: rejected observations - name: reject +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/atms_npp.yaml b/parm/atm/obs/config/atms_npp.yaml index bb668ab9d..ecddd346c 100644 --- a/parm/atm/obs/config/atms_npp.yaml +++ b/parm/atm/obs/config/atms_npp.yaml @@ -18,6 +18,7 @@ obs operator: Absorbers: [H2O,O3] Clouds: [Water, Ice] Cloud_Fraction: 1.0 + Cloud_Seeding: true obs options: Sensor_ID: &Sensor_ID atms_npp EndianType: little_endian @@ -56,7 +57,6 @@ obs bias: obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature @@ -83,6 +83,9 @@ obs pre filters: - name: InterChannelConsistency initial value: false force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false obs post filters: # Step 0-B: Calculate derived variables @@ -224,7 +227,7 @@ obs post filters: channels: *atms_npp_channels coefs: [1, -1] -# Step 0-C: Assign initial all-sky observation error +# Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -235,7 +238,7 @@ obs post filters: name: InitialObsError@DerivedMetaData channels: *atms_npp_channels -# Step 1: Remove observations from the edge of the scan +# Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: - name: brightnessTemperature @@ -250,15 +253,16 @@ obs post filters: - name: reject # Step 2: Data Thinning -#- filter: Gaussian Thinning -# horizontal_mesh: 145 -# use_reduced_horizontal_grid: true -## round_horizontal_bin_count_to_nearest: true -## partition_longitude_bins_using_mesh: true -# actions: -# - name: set -# flag: Thinning -# - name: reject +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject # Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -286,7 +290,7 @@ obs post filters: flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -338,7 +342,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: ObsErrorFactorTopo@DerivedMetaData @@ -361,7 +365,7 @@ obs post filters: name: ObsErrorFactorTopo@DerivedMetaData channels: *atms_npp_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: ObsErrorFactorTransmitTop@DerivedMetaData @@ -383,7 +387,7 @@ obs post filters: name: ObsErrorFactorTransmitTop@DerivedMetaData channels: *atms_npp_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: ObsErrorFactorSurfJacobian@DerivedMetaData @@ -409,7 +413,7 @@ obs post filters: name: ObsErrorFactorSurfJacobian@DerivedMetaData channels: *atms_npp_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: ObsErrorFactorSituDepend@DerivedMetaData @@ -504,7 +508,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -518,7 +522,7 @@ obs post filters: sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1] maxvalue: 1.0e-12 @@ -528,3 +532,24 @@ obs post filters: ignore: rejected observations - name: reject +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/testing/atms_n20.yaml b/parm/atm/obs/testing/atms_n20.yaml index 9f1c913b0..006a452f3 100644 --- a/parm/atm/obs/testing/atms_n20.yaml +++ b/parm/atm/obs/testing/atms_n20.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID atms_n20 - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: atms_n20 obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV atms_n20_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &atms_n20_channels 1-22 + geovals: filename: !ENV atms_n20_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_n20 + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV atms_n20_satbias_${GDATE}.nc4 variational bc: @@ -42,45 +45,35 @@ obs bias: obs post filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature channels: *atms_n20_channels flags: - - name: CLWRetrievalReject + - name: ScanEdgeRemoval initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - flags: - - name: HydrometeorCheckReject + force reinitialization: false + - name: Thinning initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - flags: - - name: GrossCheckReject + force reinitialization: false + - name: CLWRetrievalCheck initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - flags: - - name: InterChannelCheckReject + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency initial value: false - force reinitialization: true + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false # Step 0-B: Calculate derived variables # Calculate CLW retrieved from observation @@ -242,7 +235,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -255,7 +248,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 4: Window channel sanity check @@ -269,6 +262,10 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Variable Assignment @@ -302,7 +299,7 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject @@ -458,7 +455,6 @@ obs post filters: 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] - - filter: Background Check filter variables: - name: brightnessTemperature @@ -468,7 +464,7 @@ obs post filters: channels: *atms_n20_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject @@ -486,14 +482,37 @@ obs post filters: sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1] maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck ignore: rejected observations - name: reject + passedBenchmark: 181401 # GSI: 181403 (difference is in channel 7) diff --git a/parm/atm/obs/testing/atms_n20_noqc.yaml b/parm/atm/obs/testing/atms_n20_noqc.yaml index 1d9a9dae1..95c2463a9 100644 --- a/parm/atm/obs/testing/atms_n20_noqc.yaml +++ b/parm/atm/obs/testing/atms_n20_noqc.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID atms_n20 - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: atms_n20 obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV atms_n20_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &atms_n20_channels 1-22 + geovals: filename: !ENV atms_n20_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_n20 + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV atms_n20_satbias_${GDATE}.nc4 variational bc: diff --git a/parm/atm/obs/testing/atms_npp.yaml b/parm/atm/obs/testing/atms_npp.yaml index beae69095..c83fc7b8f 100644 --- a/parm/atm/obs/testing/atms_npp.yaml +++ b/parm/atm/obs/testing/atms_npp.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID atms_npp - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: atms_npp obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV atms_npp_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &atms_npp_channels 1-22 + geovals: filename: !ENV atms_npp_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_npp + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV atms_npp_satbias_${GDATE}.nc4 variational bc: @@ -42,45 +45,35 @@ obs bias: obs post filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature channels: *atms_npp_channels flags: - - name: CLWRetrievalReject + - name: ScanEdgeRemoval initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - flags: - - name: HydrometeorCheckReject + force reinitialization: false + - name: Thinning initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - flags: - - name: GrossCheckReject + force reinitialization: false + - name: CLWRetrievalCheck initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - flags: - - name: InterChannelCheckReject + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck initial value: false - force reinitialization: true + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false # Step 0-B: Calculate derived variables # Calculate CLW retrieved from observation @@ -242,7 +235,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -255,7 +248,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 4: Window channel sanity check @@ -269,6 +262,10 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Variable Assignment @@ -302,7 +299,7 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject @@ -467,7 +464,7 @@ obs post filters: channels: *atms_npp_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject @@ -484,15 +481,38 @@ obs post filters: use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1] maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + passedBenchmark: 182751 # GSI: 182751 diff --git a/parm/atm/obs/testing/atms_npp_noqc.yaml b/parm/atm/obs/testing/atms_npp_noqc.yaml index f8afbb175..5bbf45307 100644 --- a/parm/atm/obs/testing/atms_npp_noqc.yaml +++ b/parm/atm/obs/testing/atms_npp_noqc.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID atms_npp - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: atms_npp obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV atms_npp_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &atms_npp_channels 1-22 + geovals: filename: !ENV atms_npp_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_npp + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV atms_npp_satbias_${GDATE}.nc4 variational bc: From cd93876022a2117f0d50e838ddf30503b387ea7a Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Mon, 27 Nov 2023 16:36:42 -0600 Subject: [PATCH 37/58] Remove temporary checkout from JCSDA-internal/ufo --- CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 60ea4135c..b26b64b5b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,8 +80,7 @@ if(BUILD_GDASBUNDLE) option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" BRANCH develop ) option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) -# ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop ) - ecbuild_bundle( PROJECT ufo GIT "https://github.com/JCSDA-internal/ufo.git" BRANCH feature/mwrad ) + ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop ) # FMS and FV3 dynamical core ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) From 27ee384e0035526f264bbfca4348d38be32e9271 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Mon, 27 Nov 2023 21:28:04 -0600 Subject: [PATCH 38/58] add no python option --- .../bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml | 190 ++++++++++++++++++ .../ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml | 187 +++++++++++++++++ ush/ioda/bufr2ioda/bufr2ioda_base.py | 79 ++++++-- .../bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 20 +- ush/ioda/bufr2ioda/run_bufr2ioda.py | 13 +- 5 files changed, 463 insertions(+), 26 deletions(-) create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml b/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml new file mode 100644 index 000000000..345d4bd65 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml @@ -0,0 +1,190 @@ +# (C) Copyright 2021 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "{{ DMPDIR }}/{{ RUN }}.{{ PDY }}/{{ cyc }}/atmos/{{ RUN }}.t{{ cyc }}z.1bamua.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + + latitude: + query: "*/CLAT" + + longitude: + query: "*/CLON" + + satelliteIdentifier: + query: "*/SAID" + + satelliteInstrument: + query: "*/SIID" + + fieldOfViewNumber: + query: "*/FOVN" + + landOrSeaQualifier: + query: "*/LSQL" + + heightOfLandSurface: + query: "*/HOLS" + + heightOfStation: + query: "*/HMSL" + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + sensorScanAngle: + sensorScanAngle: + fieldOfViewNumber: "*/FOVN" + scanStart: -48.333 + scanStep: 3.333 + sensor: amsua + + sensorChannelNumber: + query: "*/BRITCSTC/CHNM" + + # ObsValue + # Note: BUFR Dump contains Antenna Temperature for all normal-feed AMSUAs except NOAA-15/16 + # NOAA-15/16 contain Brightness Temperature + antennaTemperature: + query: "*/BRITCSTC/TMBR" + + splits: + satId: + category: + variable: satelliteIdentifier + map: + _3: metop-b + _4: metop-a + _5: metop-c + _208: n17 + _209: n18 + _223: n19 + + ioda: + backend: netcdf + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.amsua.$(splitvar)_ta.tm00.nc" + + dimensions: + - name: Channel + source: variables/sensorChannelNumber + path: "*/BRITCSTC" + + globals: + - name: "platformCommonName" + type: string + value: "AMSU-A" + + - name: "platformLongDescription" + type: string + value: "MTYP 021-023 PROC AMSU-A 1B Tb" + + variables: + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteIdentifier + longName: "Satellite Identifier" + + - name: "MetaData/satelliteInstrument" + source: variables/satelliteInstrument + longName: "Satellite Instrument" + + - name: "MetaData/landOrSeaQualifier" + source: variables/landOrSeaQualifier + longName: "Land/Sea Qualifier" + + - name: "MetaData/heightOfLandSurface" + source: variables/heightOfLandSurface + longName: "Height of Land Surface" + units: "m" + + - name: "MetaData/heightOfStation" + source: variables/heightOfStation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degree" + range: [0, 180] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [0, 90] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorViewAngle" + source: variables/sensorScanAngle + longName: "Sensor View Angle" + units: "degree" + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + - name: "MetaData/sensorScanPosition" + source: variables/fieldOfViewNumber + longName: "Field of View Number" + + # ObsValue + # For now, store the antennaTemperature as brightnessTemperature + - name: "ObsValue/brightnessTemperature" + source: variables/antennaTemperature + longName: "Antenna Temperature" + units: "K" + range: [90, 380] + chunks: [1000, 15] + compressionLevel: 4 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml b/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml new file mode 100644 index 000000000..d79c3a20e --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml @@ -0,0 +1,187 @@ +# (C) Copyright 2021 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "{{ DMPDIR }}/{{ RUN }}.{{ PDY }}/{{ cyc }}/atmos/{{ RUN }}.t{{ cyc }}z.esamua.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + + latitude: + query: "*/CLATH" + + longitude: + query: "*/CLONH" + + satelliteIdentifier: + query: "*/SAID" + + satelliteAntennaCorrectionsVersionNumber: + query: "*/SACV" + + orbitNumber: + query: "*/ORBN" + + scanLineNumber: + query: "*/SLNM" + + fieldOfViewNumber: + query: "*/FOVN" + + heightOfStation: + query: "*/SELV" + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + sensorScanAngle: + sensorScanAngle: + fieldOfViewNumber: "*/FOVN" + scanStart: -48.333 + scanStep: 3.333 + sensor: amsua + + sensorChannelNumber: + query: "*/ATNCHV/INCN" + + # ObsValue + brightnessTemperature: + query: "*/ATNCHV/TMBRST" + + splits: + satId: + category: + variable: satelliteIdentifier + map: + _3: metop-b + _4: metop-a + _5: metop-c +# _206: n15 + _209: n18 + _223: n19 + + ioda: + backend: netcdf + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.esamua.$(splitvar).tm00.nc" + + dimensions: + - name: Channel + source: variables/sensorChannelNumber + path: "*/ATNCHV" + + globals: + - name: "platformCommonName" + type: string + value: "AMSUA" + + - name: "platformLongDescription" + type: string + value: "MTYP 021-033 RARS(EARS,AP,SA) AMSU-A 1C Tb DATA)" + + variables: + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteIdentifier + longName: "Satellite Identifier" + + - name: "MetaData/satelliteAntennaCorrectionsVersionNumber" + source: variables/satelliteAntennaCorrectionsVersionNumber + longName: "Satellite Antenna Corrections Version Number" + + - name: "MetaData/fieldOfViewNumber" + source: variables/fieldOfViewNumber + longName: "Field Of View Number" + + - name: "MetaData/orbitNumber" + source: variables/orbitNumber + longName: "Orbit Number" + + - name: "MetaData/scanLineNumber" + source: variables/scanLineNumber + longName: "Scan Line Number1" + + - name: "MetaData/heightOfStation" + source: variables/heightOfStation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degree" + range: [0, 180] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [0, 90] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorViewAngle" + source: variables/sensorScanAngle + longName: "Sensor View Angle" + units: "degree" + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + # ObsValue + - name: "ObsValue/brightnessTemperature" + coordinates: "longitude latitude Channel" + source: variables/brightnessTemperature + longName: "Brightness Temperature" + units: "K" + range: [100, 500] + chunks: [1000, 15] + compressionLevel: 4 diff --git a/ush/ioda/bufr2ioda/bufr2ioda_base.py b/ush/ioda/bufr2ioda/bufr2ioda_base.py index a26d7819f..516560e1e 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_base.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_base.py @@ -1,4 +1,5 @@ import json +import netCDF4 as nc import re import yaml @@ -8,21 +9,30 @@ from utils import timing_decorator +CPP = 1 +PYTHON = 2 + logger = Logger('BUFR2IODA_satwind_amv_goes.py', level='DEBUG') class Bufr2IodaBase: - def __init__(self, file_name): + def __init__(self, file_name, backend=PYTHON): with open(file_name, "r") as json_file: config = json.load(json_file) self.config = config - yaml_config = self.get_yaml_config(config) + yaml_config = self.get_yaml_config() self.yaml_config = yaml_config self.ioda_files = yaml_config['observations'][0]['ioda'].get('obsdataout') logger.info(f'Ioda output files are: {self.ioda_files}') self.splits = yaml_config['observations'][0]['obs space']['exports'].get('splits') - self.split_files = None + self.sat_ids = None + if self.splits: + self.sat_ids = self.splits['satId']['category']['map'].values() + logger.info(self.splits) + self.split_files = {} + self.yaml_path = None + self.backend = backend def check_yaml_config(yaml_config): # TODO make some consistency check? @@ -33,13 +43,38 @@ def get_attr(self, config): obs_attr = {} return obs_attr - def get_data_container(self): - pass + def get_container(self): + container = {} + if self.backend == PYTHON: + self.yaml_path = self.get_yaml_file() + container = bufr.Parser(self.yaml_path).parse() + elif self.backend == CPP: + container = {} + for sat_id in self.sat_ids: + try: + container[sat_id] = nc.Dataset(self.split_files[sat_id], 'r+') + except FileNotFoundError as e: + logger.info(e) + return container + + def get_container_variable(self, container, variable, sat_id): + ret = None + if self.backend == PYTHON: + ret = container.get(variable, sat_id) + elif self.backend == CPP: + ret = container[sat_id][variable][:] + return ret + + def replace_container_variable(self, container, variable, var, sat_id): + if self.backend == PYTHON: + container.replace(variable, sat_id) + elif self.backend == CPP: + container[sat_id][variable][:] = var def get_yaml_file(self): return self.config['yaml_file'] - def get_yaml_config(self, config): + def get_yaml_config(self): # TODO add and/or modify ymal contents from config json file. e.g. file names, etc. # What does en_bufr2ioda_yaml.py do? connection to this? yaml_file = self.get_yaml_file() @@ -50,18 +85,34 @@ def get_yaml_config(self, config): def make_split_files(self, sat_ids): # sat_id a list of ids pattern = r'{(.*?)}' - self. split_files = [re.sub(pattern, sat_id[0], self.ioda_files) for sat_id in sat_ids] + for sat_id in sat_ids: + self.split_files[sat_id] = re.sub(pattern, sat_id, self.ioda_files) - def re_map_variable(self, container, ioda_description): + def re_map_variable(self, container): # Make any changes and return for your specific case in the sub-class pass + def ioda_encode(self, container): + if self.backend == PYTHON: + ioda_description = bufr.IodaDescription(self.yaml_path) + bufr.IodaEncoder(ioda_description).encode(container) + elif self.backend == CPP: + for item in container: + container[item].close() + @timing_decorator def execute(self): - yaml_path = self.get_yaml_file() - container = bufr.Parser(yaml_path).parse() - ioda_description = bufr.IodaDescription(yaml_path) if self.splits: - self.make_split_files(container.allSubCategories()) - self.re_map_variable(container, ioda_description) - bufr.IodaEncoder(ioda_description).encode(container) + self.make_split_files(self.sat_ids) + container = self.get_container() + self.re_map_variable(container) + self.ioda_encode(container) + + def get_info(self): + if not self.split_files: + self.make_split_files(self.splits) + sat_info = {'ioda_files': self.ioda_files, + 'split': self.splits, + 'split_files': self.split_files + } + return sat_info diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index 223452424..95cf747a2 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -2,7 +2,7 @@ import argparse import os -from bufr2ioda_base import Bufr2IodaBase +from bufr2ioda_base import Bufr2IodaBase, CPP from wxflow import Logger from antcorr_application import ACCoeff, apply_ant_corr from utils import timing_decorator, nc_merge @@ -15,6 +15,8 @@ SPLIT = ('metop-a', 'metop-b', 'metop-c', 'n17', 'n18', 'n19') ioda_files = [(f'esamua.{x}.tm00.nc', f'amsua.{x}_ta.tm00.nc', f'amsua.{x}.tm00.nc') for x in SPLIT] +BACKEND = CPP + class Bufr2IodaAmusa(Bufr2IodaBase): def get_yaml_file(self): @@ -27,19 +29,18 @@ def get_yaml_file(self): return self.config['yaml_file'][1] @timing_decorator - def re_map_variable(self, container, ioda_description): + def re_map_variable(self, container): # TODO replace this follow that in GSI # read_bufrtovs.f90 # antcorr_application.f90 # search the keyword “ta2tb” for details - sat_ids = container.allSubCategories() - for sat_id in sat_ids: - ta = container.get('variables/antennaTemperature', sat_id) + for sat_id in self.sat_ids: + ta = self.get_container_variable(container, 'ObsValue/brightnessTemperature', sat_id) if ta.shape[0]: - ifov = container.get('variables/fieldOfViewNumber', sat_id) + ifov = self.get_container_variable(container, 'MetaData/sensorScanPosition', sat_id) tb = self.apply_ant_corr(sat_id, ta, ifov) - container.replace('variables/antennaTemperature', tb, sat_id) + self.replace_container_variable(container, 'ObsValue/brightnessTemperature', tb, sat_id) def apply_ant_corr(self, sat_id, ta, ifov): ac = ACCoeff() # TODO add later @@ -81,9 +82,10 @@ def merge(amsua_files): for sat_type in ['a', 'e']: print(sat_type) if sat_type == 'a': - convert = Bufr2IodaAmusa(args.config) + convert = Bufr2IodaAmusa(args.config, backend=BACKEND) else: - convert = Bufr2IodaEbmua(args.config) + convert = Bufr2IodaEbmua(args.config, backend=BACKEND) + convert.execute() amsua_files.append(convert.split_files) print(amsua_files) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 8a4a9debc..066642ab9 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -56,7 +56,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): "amsua_metop-c_v2.ACCoeff.nc" ] for fix_file in fix_files: - shutil.copy(os.path.join(config_template_dir, fix_file), os.path.join(os.getcwd(), fix_file)) + shutil.copy(os.path.join(config_template_dir, fix_file), os.path.join(DATA, fix_file)) # Specify observation types to be processed by a script BUFR_py_files = glob.glob(os.path.join(USH_IODA, 'bufr2ioda_*.py')) @@ -106,7 +106,13 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # if os.path.exists(yaml_output_file): # rm_p(yaml_output_file) - # Specify observation types to be processed by a script with combined methods + # run everything in parallel + with mp.Pool(num_cores) as pool: + pool.starmap(mp_bufr_converter, zip(exename, config_files)) + + config_files = [] + exename = [] + # Specify observation types to be processed by a script with combined methods script_type = 'bufr2ioda_combine_' BUFR_combine_files = glob.glob(os.path.join(USH_IODA, script_type + '*.py')) BUFR_combine_files = [os.path.basename(f) for f in BUFR_combine_files] @@ -114,7 +120,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): for obtype in BUFR_combine: logger.info(f"Convert {obtype}...") - json_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") + json_output_file = os.path.join(DATA, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") filename = 'bufr2ioda_' + obtype + '.json' template = os.path.join(config_template_dir, filename) gen_bufr_json(config, template, json_output_file) @@ -125,6 +131,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # append the values to the lists config_files.append(json_output_file) exename.append(bufr2iodapy) + # run everything in parallel with mp.Pool(num_cores) as pool: pool.starmap(mp_bufr_converter, zip(exename, config_files)) From 0e1ffd48814708738e7c45e4076e2857bfbd4fba Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Mon, 27 Nov 2023 22:27:47 -0600 Subject: [PATCH 39/58] Re-arrange and tidy up the YAML files for AMSU-A --- parm/atm/obs/config/amsua_metop-a.yaml | 93 +++++------- parm/atm/obs/config/amsua_metop-b.yaml | 93 +++++------- parm/atm/obs/config/amsua_metop-c.yaml | 93 +++++------- parm/atm/obs/config/amsua_n15.yaml | 52 +++++-- parm/atm/obs/config/amsua_n18.yaml | 52 +++++-- parm/atm/obs/config/amsua_n19.yaml | 54 +++++-- parm/atm/obs/testing/amsua_metop-a.yaml | 139 ++++++++++-------- parm/atm/obs/testing/amsua_metop-a_noqc.yaml | 21 +-- parm/atm/obs/testing/amsua_metop-b.yaml | 145 +++++++++++-------- parm/atm/obs/testing/amsua_metop-b_noqc.yaml | 23 +-- parm/atm/obs/testing/amsua_metop-c.yaml | 137 ++++++++++-------- parm/atm/obs/testing/amsua_metop-c_noqc.yaml | 22 +-- parm/atm/obs/testing/amsua_n15.yaml | 139 ++++++++++-------- parm/atm/obs/testing/amsua_n15_noqc.yaml | 21 +-- parm/atm/obs/testing/amsua_n18.yaml | 139 ++++++++++-------- parm/atm/obs/testing/amsua_n18_noqc.yaml | 23 +-- parm/atm/obs/testing/amsua_n19.yaml | 141 ++++++++++-------- parm/atm/obs/testing/amsua_n19_noqc.yaml | 23 +-- 18 files changed, 793 insertions(+), 617 deletions(-) diff --git a/parm/atm/obs/config/amsua_metop-a.yaml b/parm/atm/obs/config/amsua_metop-a.yaml index 34bceae43..8b7da213f 100644 --- a/parm/atm/obs/config/amsua_metop-a.yaml +++ b/parm/atm/obs/config/amsua_metop-a.yaml @@ -82,50 +82,12 @@ obs pre filters: - name: InterChannelConsistency initial value: false force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false obs post filters: -# Step 0-B: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-a_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-a_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-a_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-a_channels - flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true - -# Step 0-B: Calculate derived variables +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -254,7 +216,7 @@ obs post filters: channels: *amsua_metop-a_channels coefs: [1, -1] -# Step 0-C: Assign initial all-sky observation error +# Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -265,7 +227,7 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_metop-a_channels -# Step 1: Remove observations from the edge of the scan +# Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: - name: brightnessTemperature @@ -283,8 +245,9 @@ obs post filters: - filter: Gaussian Thinning horizontal_mesh: 145 use_reduced_horizontal_grid: true - round_horizontal_bin_count_to_nearest: true - partition_longitude_bins_using_mesh: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true actions: - name: set flag: Thinning @@ -316,7 +279,7 @@ obs post filters: flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -327,6 +290,10 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check @@ -353,7 +320,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo @@ -376,7 +343,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_metop-a_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop @@ -398,7 +365,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_metop-a_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian @@ -423,7 +390,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSurfJacobian channels: *amsua_metop-a_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend @@ -461,7 +428,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSituDepend channels: *amsua_metop-a_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -514,7 +481,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -535,3 +502,23 @@ obs post filters: flag: InterChannelConsistency ignore: rejected observations - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/amsua_metop-b.yaml b/parm/atm/obs/config/amsua_metop-b.yaml index 82d5fb950..035a36c1b 100644 --- a/parm/atm/obs/config/amsua_metop-b.yaml +++ b/parm/atm/obs/config/amsua_metop-b.yaml @@ -82,50 +82,12 @@ obs pre filters: - name: InterChannelConsistency initial value: false force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false obs post filters: -# Step 0-B: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-b_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-b_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-b_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-b_channels - flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true - -# Step 0-B: Calculate derived variables +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -254,7 +216,7 @@ obs post filters: channels: *amsua_metop-b_channels coefs: [1, -1] -# Step 0-C: Assign initial all-sky observation error +# Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -265,7 +227,7 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_metop-b_channels -# Step 1: Remove observations from the edge of the scan +# Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: - name: brightnessTemperature @@ -283,8 +245,9 @@ obs post filters: - filter: Gaussian Thinning horizontal_mesh: 145 use_reduced_horizontal_grid: true - round_horizontal_bin_count_to_nearest: true - partition_longitude_bins_using_mesh: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true actions: - name: set flag: Thinning @@ -316,7 +279,7 @@ obs post filters: flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -327,6 +290,10 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check @@ -353,7 +320,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo @@ -376,7 +343,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_metop-b_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop @@ -398,7 +365,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_metop-b_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian @@ -423,7 +390,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSurfJacobian channels: *amsua_metop-b_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend @@ -461,7 +428,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSituDepend channels: *amsua_metop-b_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -514,7 +481,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -535,3 +502,23 @@ obs post filters: flag: InterChannelConsistency ignore: rejected observations - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + use_flag: [ -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, + 1, 1, 1, 1, -1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/amsua_metop-c.yaml b/parm/atm/obs/config/amsua_metop-c.yaml index 818fd6d73..3dceb20ed 100644 --- a/parm/atm/obs/config/amsua_metop-c.yaml +++ b/parm/atm/obs/config/amsua_metop-c.yaml @@ -82,50 +82,12 @@ obs pre filters: - name: InterChannelConsistency initial value: false force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false obs post filters: -# Step 0-B: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-c_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-c_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-c_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-c_channels - flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true - -# Step 0-B: Calculate derived variables +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -254,7 +216,7 @@ obs post filters: channels: *amsua_metop-c_channels coefs: [1, -1] -# Step 0-C: Assign initial all-sky observation error +# Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -265,7 +227,7 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_metop-c_channels -# Step 1: Remove observations from the edge of the scan +# Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: - name: brightnessTemperature @@ -283,8 +245,9 @@ obs post filters: - filter: Gaussian Thinning horizontal_mesh: 145 use_reduced_horizontal_grid: true - round_horizontal_bin_count_to_nearest: true - partition_longitude_bins_using_mesh: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true actions: - name: set flag: Thinning @@ -316,7 +279,7 @@ obs post filters: flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -327,6 +290,10 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check @@ -353,7 +320,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo @@ -376,7 +343,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_metop-c_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop @@ -398,7 +365,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_metop-c_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian @@ -423,7 +390,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSurfJacobian channels: *amsua_metop-c_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend @@ -461,7 +428,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSituDepend channels: *amsua_metop-c_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -514,7 +481,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -535,3 +502,23 @@ obs post filters: flag: InterChannelConsistency ignore: rejected observations - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/amsua_n15.yaml b/parm/atm/obs/config/amsua_n15.yaml index 511d7860a..2d191a38a 100644 --- a/parm/atm/obs/config/amsua_n15.yaml +++ b/parm/atm/obs/config/amsua_n15.yaml @@ -82,9 +82,12 @@ obs pre filters: - name: InterChannelConsistency initial value: false force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false obs post filters: -# Step 0-B: Calculate derived variables +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -213,7 +216,7 @@ obs post filters: channels: *amsua_n15_channels coefs: [1, -1] -# Step 0-C: Assign initial all-sky observation error +# Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -224,7 +227,7 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_n15_channels -# Step 1: Remove observations from the edge of the scan +# Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: - name: brightnessTemperature @@ -242,8 +245,9 @@ obs post filters: - filter: Gaussian Thinning horizontal_mesh: 145 use_reduced_horizontal_grid: true - round_horizontal_bin_count_to_nearest: true - partition_longitude_bins_using_mesh: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true actions: - name: set flag: Thinning @@ -275,7 +279,7 @@ obs post filters: flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -286,6 +290,10 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check @@ -312,7 +320,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo @@ -335,7 +343,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_n15_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop @@ -357,7 +365,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_n15_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian @@ -382,7 +390,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSurfJacobian channels: *amsua_n15_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend @@ -420,7 +428,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSituDepend channels: *amsua_n15_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -473,7 +481,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -494,3 +502,23 @@ obs post filters: flag: InterChannelConsistency ignore: rejected observations - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + use_flag: [ 1, 1, 1, 1, 1, + -1, 1, 1, 1, 1, + -1, 1, 1, -1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/amsua_n18.yaml b/parm/atm/obs/config/amsua_n18.yaml index 7d749c4bc..c5673e062 100644 --- a/parm/atm/obs/config/amsua_n18.yaml +++ b/parm/atm/obs/config/amsua_n18.yaml @@ -82,9 +82,12 @@ obs pre filters: - name: InterChannelConsistency initial value: false force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false obs post filters: -# Step 0-B: Calculate derived variables +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -213,7 +216,7 @@ obs post filters: channels: *amsua_n18_channels coefs: [1, -1] -# Step 0-C: Assign initial all-sky observation error +# Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -224,7 +227,7 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_n18_channels -# Step 1: Remove observations from the edge of the scan +# Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: - name: brightnessTemperature @@ -242,8 +245,9 @@ obs post filters: - filter: Gaussian Thinning horizontal_mesh: 145 use_reduced_horizontal_grid: true - round_horizontal_bin_count_to_nearest: true - partition_longitude_bins_using_mesh: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true actions: - name: set flag: Thinning @@ -275,7 +279,7 @@ obs post filters: flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -286,6 +290,10 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check @@ -312,7 +320,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo @@ -335,7 +343,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_n18_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop @@ -357,7 +365,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_n18_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian @@ -382,7 +390,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSurfJacobian channels: *amsua_n18_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend @@ -420,7 +428,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSituDepend channels: *amsua_n18_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -473,7 +481,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -494,3 +502,23 @@ obs post filters: flag: InterChannelConsistency ignore: rejected observations - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + use_flag: [ 1, 1, 1, 1, -1, + 1, 1, -1, -1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/amsua_n19.yaml b/parm/atm/obs/config/amsua_n19.yaml index a940447d8..b6c64ae08 100644 --- a/parm/atm/obs/config/amsua_n19.yaml +++ b/parm/atm/obs/config/amsua_n19.yaml @@ -82,9 +82,12 @@ obs pre filters: - name: InterChannelConsistency initial value: false force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false obs post filters: -# Step 0-B: Calculate derived variables +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -213,7 +216,7 @@ obs post filters: channels: *amsua_n19_channels coefs: [1, -1] -# Step 0-C: Assign initial all-sky observation error +# Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -224,7 +227,7 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_n19_channels -# Step 1: Remove observations from the edge of the scan +# Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: - name: brightnessTemperature @@ -242,8 +245,9 @@ obs post filters: - filter: Gaussian Thinning horizontal_mesh: 145 use_reduced_horizontal_grid: true - round_horizontal_bin_count_to_nearest: true - partition_longitude_bins_using_mesh: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true actions: - name: set flag: Thinning @@ -275,17 +279,21 @@ obs post filters: flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature channels: 1-6, 15 test variables: - name: DerivedMetaData/Innovation - channels: 1, 2 4-6, 15 + channels: 1, 2, 4-6, 15 maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check @@ -312,7 +320,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo @@ -335,7 +343,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_n19_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop @@ -357,7 +365,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_n19_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian @@ -382,7 +390,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSurfJacobian channels: *amsua_n19_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend @@ -420,7 +428,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSituDepend channels: *amsua_n19_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -473,7 +481,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -494,3 +502,23 @@ obs post filters: flag: InterChannelConsistency ignore: rejected observations - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/testing/amsua_metop-a.yaml b/parm/atm/obs/testing/amsua_metop-a.yaml index f7009b514..fabed5ee0 100644 --- a/parm/atm/obs/testing/amsua_metop-a.yaml +++ b/parm/atm/obs/testing/amsua_metop-a.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_metop-a - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_metop-a obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV amsua_metop-a_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &amsua_metop-a_channels 1-15 + geovals: filename: !ENV amsua_metop-a_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-a + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_metop-a_satbias_${GDATE}.nc4 variational bc: @@ -42,47 +45,37 @@ obs bias: obs post filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-a_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-a_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-a_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature + filter variables: + - name: brightnessTemperature channels: *amsua_metop-a_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false -# Step 0-B: Calculate derived variables +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -211,7 +204,7 @@ obs post filters: channels: *amsua_metop-a_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 1: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -232,7 +225,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -245,10 +238,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,6 +252,10 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check @@ -281,11 +278,11 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo @@ -308,7 +305,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_metop-a_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop @@ -330,7 +327,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_metop-a_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian @@ -355,7 +352,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSurfJacobian channels: *amsua_metop-a_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend @@ -393,7 +390,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSituDepend channels: *amsua_metop-a_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -442,11 +439,11 @@ obs post filters: channels: *amsua_metop-a_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -464,8 +461,30 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject -passedBenchmark: 91979 + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + +passedBenchmark: 91978 +#passedBenchmark: 91978 # no useflag check diff --git a/parm/atm/obs/testing/amsua_metop-a_noqc.yaml b/parm/atm/obs/testing/amsua_metop-a_noqc.yaml index a19c44833..f029fd8ab 100644 --- a/parm/atm/obs/testing/amsua_metop-a_noqc.yaml +++ b/parm/atm/obs/testing/amsua_metop-a_noqc.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_metop-a - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_metop-a obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV amsua_metop-a_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &amsua_metop-a_channels 1-15 + geovals: filename: !ENV amsua_metop-a_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-a + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_metop-a_satbias_${GDATE}.nc4 variational bc: diff --git a/parm/atm/obs/testing/amsua_metop-b.yaml b/parm/atm/obs/testing/amsua_metop-b.yaml index ebd1f3999..8c2ac5501 100644 --- a/parm/atm/obs/testing/amsua_metop-b.yaml +++ b/parm/atm/obs/testing/amsua_metop-b.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_metop-b - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_metop-b obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV amsua_metop-b_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &amsua_metop-b_channels 1-15 + geovals: filename: !ENV amsua_metop-b_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-b + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_metop-b_satbias_${GDATE}.nc4 variational bc: @@ -42,47 +45,37 @@ obs bias: obs post filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-b_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-b_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-b_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature + filter variables: + - name: brightnessTemperature channels: *amsua_metop-b_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false -# Step 0-B: Calculate derived variables +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -211,7 +204,7 @@ obs post filters: channels: *amsua_metop-b_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 1: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -232,7 +225,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -245,10 +238,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,6 +252,10 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check @@ -281,11 +278,11 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo @@ -308,7 +305,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_metop-b_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop @@ -330,7 +327,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_metop-b_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian @@ -355,7 +352,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSurfJacobian channels: *amsua_metop-b_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend @@ -393,7 +390,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSituDepend channels: *amsua_metop-b_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -442,11 +439,11 @@ obs post filters: channels: *amsua_metop-b_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -458,14 +455,36 @@ obs post filters: channels: *amsua_metop-b_channels use passive_bc: true sensor: *Sensor_ID - use_flag: [ -1, -1, -1, -1, -1, - -1, -1, 1, 1, 1, - 1, 1, 1, 1, -1 ] + use_flag: [ -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, + 1, 1, 1, 1, -1 ] maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject -passedBenchmark: 70590 + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + use_flag: [ -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, + 1, 1, 1, 1, -1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + +passedBenchmark: 62069 +#passedBenchmark: 70974 # no useflag check diff --git a/parm/atm/obs/testing/amsua_metop-b_noqc.yaml b/parm/atm/obs/testing/amsua_metop-b_noqc.yaml index adc4b9395..4b5aa7cbb 100644 --- a/parm/atm/obs/testing/amsua_metop-b_noqc.yaml +++ b/parm/atm/obs/testing/amsua_metop-b_noqc.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_metop-b - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_metop-b obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV amsua_metop-b_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &amsua_metop-b_channels 1-15 + geovals: filename: !ENV amsua_metop-b_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-b + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_metop-b_satbias_${GDATE}.nc4 variational bc: @@ -132,6 +135,6 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_metop-b_channels -passedBenchmark: 124549 +passedBenchmark: 125148 #vector ref: GsiHofXBc #tolerance: 1.e-7 diff --git a/parm/atm/obs/testing/amsua_metop-c.yaml b/parm/atm/obs/testing/amsua_metop-c.yaml index 49871a8af..6e0b438b2 100644 --- a/parm/atm/obs/testing/amsua_metop-c.yaml +++ b/parm/atm/obs/testing/amsua_metop-c.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_metop-c - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_metop-c obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV amsua_metop-c_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &amsua_metop-c_channels 1-15 + geovals: filename: !ENV amsua_metop-c_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-c + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_metop-c_satbias_${GDATE}.nc4 variational bc: @@ -42,47 +45,37 @@ obs bias: obs post filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-c_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-c_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_metop-c_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature + filter variables: + - name: brightnessTemperature channels: *amsua_metop-c_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false -# Step 0-B: Calculate derived variables +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -211,7 +204,7 @@ obs post filters: channels: *amsua_metop-c_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 1: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -232,7 +225,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -245,10 +238,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,6 +252,10 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check @@ -281,11 +278,11 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo @@ -308,7 +305,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_metop-c_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop @@ -330,7 +327,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_metop-c_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian @@ -355,7 +352,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSurfJacobian channels: *amsua_metop-c_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend @@ -393,7 +390,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSituDepend channels: *amsua_metop-c_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -442,11 +439,11 @@ obs post filters: channels: *amsua_metop-c_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -464,7 +461,29 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + passedBenchmark: 112377 +#passedBenchmark: 112377 # no useflag check diff --git a/parm/atm/obs/testing/amsua_metop-c_noqc.yaml b/parm/atm/obs/testing/amsua_metop-c_noqc.yaml index 885f3b38f..9624b230c 100644 --- a/parm/atm/obs/testing/amsua_metop-c_noqc.yaml +++ b/parm/atm/obs/testing/amsua_metop-c_noqc.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_metop-c - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_metop-c obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV amsua_metop-c_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &amsua_metop-c_channels 1-15 + geovals: filename: !ENV amsua_metop-c_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-c + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_metop-c_satbias_${GDATE}.nc4 variational bc: @@ -135,4 +138,3 @@ obs post filters: passedBenchmark: 133601 #vector ref: GsiHofXBc #tolerance: 1.e-7 - diff --git a/parm/atm/obs/testing/amsua_n15.yaml b/parm/atm/obs/testing/amsua_n15.yaml index 1afd7eb77..5c4ba4c4c 100644 --- a/parm/atm/obs/testing/amsua_n15.yaml +++ b/parm/atm/obs/testing/amsua_n15.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_n15 - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_n15 obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV amsua_n15_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &amsua_n15_channels 1-15 + geovals: filename: !ENV amsua_n15_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n15 + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_n15_satbias_${GDATE}.nc4 variational bc: @@ -42,47 +45,37 @@ obs bias: obs post filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n15_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n15_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n15_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature + filter variables: + - name: brightnessTemperature channels: *amsua_n15_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false -# Step 0-B: Calculate derived variables +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -211,7 +204,7 @@ obs post filters: channels: *amsua_n15_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 1: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -232,7 +225,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -245,10 +238,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,6 +252,10 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check @@ -281,11 +278,11 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo @@ -308,7 +305,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_n15_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop @@ -330,7 +327,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_n15_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian @@ -355,7 +352,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSurfJacobian channels: *amsua_n15_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend @@ -393,7 +390,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSituDepend channels: *amsua_n15_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -442,11 +439,11 @@ obs post filters: channels: *amsua_n15_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -464,7 +461,29 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject -passedBenchmark: 85704 + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + use_flag: [ 1, 1, 1, 1, 1, + -1, 1, 1, 1, 1, + -1, 1, 1, -1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + +passedBenchmark: 80770 +#passedBenchmark: 85704 # no useflag check diff --git a/parm/atm/obs/testing/amsua_n15_noqc.yaml b/parm/atm/obs/testing/amsua_n15_noqc.yaml index a16d1aed7..c94d3ddb2 100644 --- a/parm/atm/obs/testing/amsua_n15_noqc.yaml +++ b/parm/atm/obs/testing/amsua_n15_noqc.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_n15 - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_n15 obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV amsua_n15_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &amsua_n15_channels 1-15 + geovals: filename: !ENV amsua_n15_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n15 + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_n15_satbias_${GDATE}.nc4 variational bc: diff --git a/parm/atm/obs/testing/amsua_n18.yaml b/parm/atm/obs/testing/amsua_n18.yaml index 9dff0330c..b9654cf8e 100644 --- a/parm/atm/obs/testing/amsua_n18.yaml +++ b/parm/atm/obs/testing/amsua_n18.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_n18 - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_n18 obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV amsua_n18_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &amsua_n18_channels 1-15 + geovals: filename: !ENV amsua_n18_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n18 + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_n18_satbias_${GDATE}.nc4 variational bc: @@ -42,47 +45,37 @@ obs bias: obs post filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n18_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n18_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n18_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature + filter variables: + - name: brightnessTemperature channels: *amsua_n18_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false -# Step 0-B: Calculate derived variables +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -211,7 +204,7 @@ obs post filters: channels: *amsua_n18_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 1: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -232,7 +225,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -245,10 +238,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,6 +252,10 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check @@ -281,11 +278,11 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo @@ -308,7 +305,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_n18_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop @@ -330,7 +327,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_n18_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian @@ -355,7 +352,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSurfJacobian channels: *amsua_n18_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend @@ -393,7 +390,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSituDepend channels: *amsua_n18_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -442,11 +439,11 @@ obs post filters: channels: *amsua_n18_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -464,8 +461,30 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject -passedBenchmark: 115823 + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + use_flag: [ 1, 1, 1, 1, -1, + 1, 1, -1, -1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + +passedBenchmark: 92176 +#passedBenchmark: 115991 # no useflag check diff --git a/parm/atm/obs/testing/amsua_n18_noqc.yaml b/parm/atm/obs/testing/amsua_n18_noqc.yaml index 5bee14388..597ab563d 100644 --- a/parm/atm/obs/testing/amsua_n18_noqc.yaml +++ b/parm/atm/obs/testing/amsua_n18_noqc.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_n18 - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_n18 obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV amsua_n18_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &amsua_n18_channels 1-15 + geovals: filename: !ENV amsua_n18_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n18 + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_n18_satbias_${GDATE}.nc4 variational bc: @@ -132,6 +135,6 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_n18_channels -passedBenchmark: 137130 +passedBenchmark: 137310 #vector ref: GsiHofXBc #tolerance: 1.e-7 diff --git a/parm/atm/obs/testing/amsua_n19.yaml b/parm/atm/obs/testing/amsua_n19.yaml index 9eff67593..36288ab3e 100644 --- a/parm/atm/obs/testing/amsua_n19.yaml +++ b/parm/atm/obs/testing/amsua_n19.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_n19 - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_n19 obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV amsua_n19_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &amsua_n19_channels 1-15 + geovals: filename: !ENV amsua_n19_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n19 + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_n19_satbias_${GDATE}.nc4 variational bc: @@ -42,47 +45,37 @@ obs bias: obs post filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature + filter variables: + - name: brightnessTemperature channels: *amsua_n19_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false -# Step 0-B: Calculate derived variables +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -211,7 +204,7 @@ obs post filters: channels: *amsua_n19_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 1: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -232,7 +225,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -245,20 +238,24 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature channels: 1-6, 15 test variables: - name: DerivedMetaData/Innovation - channels: 1, 2 4-6, 15 + channels: 1, 2, 4-6, 15 maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check @@ -281,11 +278,11 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo @@ -308,7 +305,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_n19_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop @@ -330,7 +327,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_n19_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian @@ -355,7 +352,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSurfJacobian channels: *amsua_n19_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend @@ -393,7 +390,7 @@ obs post filters: name: DerivedMetaData/ObsErrorFactorSituDepend channels: *amsua_n19_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -442,11 +439,11 @@ obs post filters: channels: *amsua_n19_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -464,7 +461,29 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject -passedBenchmark: 106461 + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + +passedBenchmark: 94073 +#passedBenchmark: 106514 # no use flag check diff --git a/parm/atm/obs/testing/amsua_n19_noqc.yaml b/parm/atm/obs/testing/amsua_n19_noqc.yaml index b8ddd2b86..98c7efed5 100644 --- a/parm/atm/obs/testing/amsua_n19_noqc.yaml +++ b/parm/atm/obs/testing/amsua_n19_noqc.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_n19 - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_n19 obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV amsua_n19_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &amsua_n19_channels 1-15 + geovals: filename: !ENV amsua_n19_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n19 + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_n19_satbias_${GDATE}.nc4 variational bc: @@ -132,6 +135,6 @@ obs post filters: name: DerivedMetaData/InitialObsError channels: *amsua_n19_channels -passedBenchmark: 136652 +passedBenchmark: 136742 #vector ref: GsiHofXBc #tolerance: 1.e-7 From 266e1537646da9ed6419c7e76e58fc800f02a963 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Mon, 27 Nov 2023 22:42:37 -0600 Subject: [PATCH 40/58] change post filters to pre filters for diagnostic flags --- parm/atm/obs/testing/amsua_metop-a.yaml | 3 ++- parm/atm/obs/testing/amsua_metop-b.yaml | 3 ++- parm/atm/obs/testing/amsua_metop-c.yaml | 3 ++- parm/atm/obs/testing/amsua_n15.yaml | 3 ++- parm/atm/obs/testing/amsua_n18.yaml | 3 ++- parm/atm/obs/testing/amsua_n19.yaml | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/parm/atm/obs/testing/amsua_metop-a.yaml b/parm/atm/obs/testing/amsua_metop-a.yaml index fabed5ee0..2f6dc5696 100644 --- a/parm/atm/obs/testing/amsua_metop-a.yaml +++ b/parm/atm/obs/testing/amsua_metop-a.yaml @@ -43,7 +43,7 @@ obs bias: order: 2 - name: scan_angle -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags - filter: Create Diagnostic Flags filter variables: @@ -75,6 +75,7 @@ obs post filters: initial value: false force reinitialization: false +obs post filters: # Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment diff --git a/parm/atm/obs/testing/amsua_metop-b.yaml b/parm/atm/obs/testing/amsua_metop-b.yaml index 8c2ac5501..0890c213a 100644 --- a/parm/atm/obs/testing/amsua_metop-b.yaml +++ b/parm/atm/obs/testing/amsua_metop-b.yaml @@ -43,7 +43,7 @@ obs bias: order: 2 - name: scan_angle -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags - filter: Create Diagnostic Flags filter variables: @@ -75,6 +75,7 @@ obs post filters: initial value: false force reinitialization: false +obs post filters: # Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment diff --git a/parm/atm/obs/testing/amsua_metop-c.yaml b/parm/atm/obs/testing/amsua_metop-c.yaml index 6e0b438b2..cc8bb7330 100644 --- a/parm/atm/obs/testing/amsua_metop-c.yaml +++ b/parm/atm/obs/testing/amsua_metop-c.yaml @@ -43,7 +43,7 @@ obs bias: order: 2 - name: scan_angle -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags - filter: Create Diagnostic Flags filter variables: @@ -75,6 +75,7 @@ obs post filters: initial value: false force reinitialization: false +obs post filters: # Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment diff --git a/parm/atm/obs/testing/amsua_n15.yaml b/parm/atm/obs/testing/amsua_n15.yaml index 5c4ba4c4c..3bd1c0932 100644 --- a/parm/atm/obs/testing/amsua_n15.yaml +++ b/parm/atm/obs/testing/amsua_n15.yaml @@ -43,7 +43,7 @@ obs bias: order: 2 - name: scan_angle -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags - filter: Create Diagnostic Flags filter variables: @@ -75,6 +75,7 @@ obs post filters: initial value: false force reinitialization: false +obs post filters: # Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment diff --git a/parm/atm/obs/testing/amsua_n18.yaml b/parm/atm/obs/testing/amsua_n18.yaml index b9654cf8e..33206b14d 100644 --- a/parm/atm/obs/testing/amsua_n18.yaml +++ b/parm/atm/obs/testing/amsua_n18.yaml @@ -43,7 +43,7 @@ obs bias: order: 2 - name: scan_angle -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags - filter: Create Diagnostic Flags filter variables: @@ -75,6 +75,7 @@ obs post filters: initial value: false force reinitialization: false +obs post filters: # Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment diff --git a/parm/atm/obs/testing/amsua_n19.yaml b/parm/atm/obs/testing/amsua_n19.yaml index 36288ab3e..b0f3bb45e 100644 --- a/parm/atm/obs/testing/amsua_n19.yaml +++ b/parm/atm/obs/testing/amsua_n19.yaml @@ -43,7 +43,7 @@ obs bias: order: 2 - name: scan_angle -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags - filter: Create Diagnostic Flags filter variables: @@ -75,6 +75,7 @@ obs post filters: initial value: false force reinitialization: false +obs post filters: # Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment From 56bfda25e4a4d1f4a8ace9af8aa46358793d946a Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 28 Nov 2023 08:52:08 -0500 Subject: [PATCH 41/58] Revert 3dvar_dripcg.yaml --- parm/atm/variational/3dvar_dripcg.yaml | 36 +++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml index 2f9ccb76f..abd945842 100644 --- a/parm/atm/variational/3dvar_dripcg.yaml +++ b/parm/atm/variational/3dvar_dripcg.yaml @@ -33,7 +33,7 @@ variational: minimizer: algorithm: DRIPCG iterations: - - ninner: 1 + - ninner: 2 gradient norm reduction: 1e-10 test: on geometry: @@ -50,6 +50,40 @@ variational: field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml diagnostics: departures: bkgmob + - ninner: 4 + gradient norm reduction: 1e-10 + test: on + geometry: + fms initialization: + namelist filename: ./fv3jedi/fmsmpp.nml + field table filename: ./fv3jedi/field_table + akbk: ./fv3jedi/akbk.nc4 + layout: + - $(layout_x) + - $(layout_y) + npx: $(npx_anl) + npy: $(npy_anl) + npz: $(npz_anl) + field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml + diagnostics: + departures: bkgmob1 final: diagnostics: departures: anlmob + increment: + output: + filetype: auxgrid + gridtype: gaussian + filename: ./anl/atminc. + geometry: + fms initialization: + namelist filename: ./fv3jedi/fmsmpp.nml + field table filename: ./fv3jedi/field_table + akbk: ./fv3jedi/akbk.nc4 + layout: + - $(layout_x) + - $(layout_y) + npx: $(npx_anl) + npy: $(npy_anl) + npz: $(npz_anl) + field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml From 94cec744ed3570baa7a8f4833d0d0a35159cbcf5 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Tue, 28 Nov 2023 19:22:28 -0600 Subject: [PATCH 42/58] a new version of amsua converter --- CMakeLists.txt | 2 +- .../bufr2ioda_combine_ncep_1bamua_ta.yml | 190 ------------------ .../bufr2ioda_combine_ncep_esamua.yml | 187 ----------------- .../bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml | 2 +- .../ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml | 2 +- ush/ioda/bufr2ioda/bufr2ioda_base.py | 43 ++-- .../bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 46 +++-- ush/ioda/bufr2ioda/run_bufr2ioda.py | 2 +- ush/ioda/bufr2ioda/utils.py | 11 +- 9 files changed, 64 insertions(+), 421 deletions(-) delete mode 100644 parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_1bamua_ta.yml delete mode 100644 parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_esamua.yml diff --git a/CMakeLists.txt b/CMakeLists.txt index dfee7c7ec..6bed0bba3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,7 +107,7 @@ if(BUILD_GDASBUNDLE) # Build IODA converters option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON) if(BUILD_IODA_CONVERTERS) - ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" ) + ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" BRANCH develop) endif() # Land associated repositories diff --git a/parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_1bamua_ta.yml b/parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_1bamua_ta.yml deleted file mode 100644 index 345d4bd65..000000000 --- a/parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_1bamua_ta.yml +++ /dev/null @@ -1,190 +0,0 @@ -# (C) Copyright 2021 NOAA/NWS/NCEP/EMC -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -observations: - - obs space: - name: bufr - obsdatain: "{{ DMPDIR }}/{{ RUN }}.{{ PDY }}/{{ cyc }}/atmos/{{ RUN }}.t{{ cyc }}z.1bamua.tm00.bufr_d" - - exports: - variables: - # MetaData - timestamp: - datetime: - year: "*/YEAR" - month: "*/MNTH" - day: "*/DAYS" - hour: "*/HOUR" - minute: "*/MINU" - - latitude: - query: "*/CLAT" - - longitude: - query: "*/CLON" - - satelliteIdentifier: - query: "*/SAID" - - satelliteInstrument: - query: "*/SIID" - - fieldOfViewNumber: - query: "*/FOVN" - - landOrSeaQualifier: - query: "*/LSQL" - - heightOfLandSurface: - query: "*/HOLS" - - heightOfStation: - query: "*/HMSL" - - solarZenithAngle: - query: "*/SOZA" - - solarAzimuthAngle: - query: "*/SOLAZI" - - sensorZenithAngle: - query: "*/SAZA" - - sensorAzimuthAngle: - query: "*/BEARAZ" - - sensorScanAngle: - sensorScanAngle: - fieldOfViewNumber: "*/FOVN" - scanStart: -48.333 - scanStep: 3.333 - sensor: amsua - - sensorChannelNumber: - query: "*/BRITCSTC/CHNM" - - # ObsValue - # Note: BUFR Dump contains Antenna Temperature for all normal-feed AMSUAs except NOAA-15/16 - # NOAA-15/16 contain Brightness Temperature - antennaTemperature: - query: "*/BRITCSTC/TMBR" - - splits: - satId: - category: - variable: satelliteIdentifier - map: - _3: metop-b - _4: metop-a - _5: metop-c - _208: n17 - _209: n18 - _223: n19 - - ioda: - backend: netcdf - obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.amsua.$(splitvar)_ta.tm00.nc" - - dimensions: - - name: Channel - source: variables/sensorChannelNumber - path: "*/BRITCSTC" - - globals: - - name: "platformCommonName" - type: string - value: "AMSU-A" - - - name: "platformLongDescription" - type: string - value: "MTYP 021-023 PROC AMSU-A 1B Tb" - - variables: - # MetaData - - name: "MetaData/dateTime" - source: variables/timestamp - longName: "Datetime" - units: "seconds since 1970-01-01T00:00:00Z" - - - name: "MetaData/latitude" - source: variables/latitude - longName: "Latitude" - units: "degree_north" - range: [-90, 90] - - - name: "MetaData/longitude" - source: variables/longitude - longName: "Longitude" - units: "degree_east" - range: [-180, 180] - - - name: "MetaData/satelliteIdentifier" - source: variables/satelliteIdentifier - longName: "Satellite Identifier" - - - name: "MetaData/satelliteInstrument" - source: variables/satelliteInstrument - longName: "Satellite Instrument" - - - name: "MetaData/landOrSeaQualifier" - source: variables/landOrSeaQualifier - longName: "Land/Sea Qualifier" - - - name: "MetaData/heightOfLandSurface" - source: variables/heightOfLandSurface - longName: "Height of Land Surface" - units: "m" - - - name: "MetaData/heightOfStation" - source: variables/heightOfStation - longName: "Altitude of Satellite" - units: "m" - - - name: "MetaData/solarZenithAngle" - source: variables/solarZenithAngle - longName: "Solar Zenith Angle" - units: "degree" - range: [0, 180] - - - name: "MetaData/solarAzimuthAngle" - source: variables/solarAzimuthAngle - longName: "Solar Azimuth Angle" - units: "degree" - range: [0, 360] - - - name: "MetaData/sensorZenithAngle" - source: variables/sensorZenithAngle - longName: "Sensor Zenith Angle" - units: "degree" - range: [0, 90] - - - name: "MetaData/sensorAzimuthAngle" - source: variables/sensorAzimuthAngle - longName: "Sensor Azimuth Angle" - units: "degree" - range: [0, 360] - - - name: "MetaData/sensorViewAngle" - source: variables/sensorScanAngle - longName: "Sensor View Angle" - units: "degree" - - - name: "MetaData/sensorChannelNumber" - source: variables/sensorChannelNumber - longName: "Sensor Channel Number" - - - name: "MetaData/sensorScanPosition" - source: variables/fieldOfViewNumber - longName: "Field of View Number" - - # ObsValue - # For now, store the antennaTemperature as brightnessTemperature - - name: "ObsValue/brightnessTemperature" - source: variables/antennaTemperature - longName: "Antenna Temperature" - units: "K" - range: [90, 380] - chunks: [1000, 15] - compressionLevel: 4 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_esamua.yml b/parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_esamua.yml deleted file mode 100644 index d79c3a20e..000000000 --- a/parm/ioda/bufr2ioda/bufr2ioda_combine_ncep_esamua.yml +++ /dev/null @@ -1,187 +0,0 @@ -# (C) Copyright 2021 NOAA/NWS/NCEP/EMC -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -observations: - - obs space: - name: bufr - obsdatain: "{{ DMPDIR }}/{{ RUN }}.{{ PDY }}/{{ cyc }}/atmos/{{ RUN }}.t{{ cyc }}z.esamua.tm00.bufr_d" - - exports: - variables: - # MetaData - timestamp: - datetime: - year: "*/YEAR" - month: "*/MNTH" - day: "*/DAYS" - hour: "*/HOUR" - minute: "*/MINU" - - latitude: - query: "*/CLATH" - - longitude: - query: "*/CLONH" - - satelliteIdentifier: - query: "*/SAID" - - satelliteAntennaCorrectionsVersionNumber: - query: "*/SACV" - - orbitNumber: - query: "*/ORBN" - - scanLineNumber: - query: "*/SLNM" - - fieldOfViewNumber: - query: "*/FOVN" - - heightOfStation: - query: "*/SELV" - - solarZenithAngle: - query: "*/SOZA" - - solarAzimuthAngle: - query: "*/SOLAZI" - - sensorZenithAngle: - query: "*/SAZA" - - sensorAzimuthAngle: - query: "*/BEARAZ" - - sensorScanAngle: - sensorScanAngle: - fieldOfViewNumber: "*/FOVN" - scanStart: -48.333 - scanStep: 3.333 - sensor: amsua - - sensorChannelNumber: - query: "*/ATNCHV/INCN" - - # ObsValue - brightnessTemperature: - query: "*/ATNCHV/TMBRST" - - splits: - satId: - category: - variable: satelliteIdentifier - map: - _3: metop-b - _4: metop-a - _5: metop-c -# _206: n15 - _209: n18 - _223: n19 - - ioda: - backend: netcdf - obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.esamua.$(splitvar).tm00.nc" - - dimensions: - - name: Channel - source: variables/sensorChannelNumber - path: "*/ATNCHV" - - globals: - - name: "platformCommonName" - type: string - value: "AMSUA" - - - name: "platformLongDescription" - type: string - value: "MTYP 021-033 RARS(EARS,AP,SA) AMSU-A 1C Tb DATA)" - - variables: - # MetaData - - name: "MetaData/dateTime" - source: variables/timestamp - longName: "Datetime" - units: "seconds since 1970-01-01T00:00:00Z" - - - name: "MetaData/latitude" - source: variables/latitude - longName: "Latitude" - units: "degree_north" - range: [-90, 90] - - - name: "MetaData/longitude" - source: variables/longitude - longName: "Longitude" - units: "degree_east" - range: [-180, 180] - - - name: "MetaData/satelliteIdentifier" - source: variables/satelliteIdentifier - longName: "Satellite Identifier" - - - name: "MetaData/satelliteAntennaCorrectionsVersionNumber" - source: variables/satelliteAntennaCorrectionsVersionNumber - longName: "Satellite Antenna Corrections Version Number" - - - name: "MetaData/fieldOfViewNumber" - source: variables/fieldOfViewNumber - longName: "Field Of View Number" - - - name: "MetaData/orbitNumber" - source: variables/orbitNumber - longName: "Orbit Number" - - - name: "MetaData/scanLineNumber" - source: variables/scanLineNumber - longName: "Scan Line Number1" - - - name: "MetaData/heightOfStation" - source: variables/heightOfStation - longName: "Altitude of Satellite" - units: "m" - - - name: "MetaData/solarZenithAngle" - source: variables/solarZenithAngle - longName: "Solar Zenith Angle" - units: "degree" - range: [0, 180] - - - name: "MetaData/solarAzimuthAngle" - source: variables/solarAzimuthAngle - longName: "Solar Azimuth Angle" - units: "degree" - range: [0, 360] - - - name: "MetaData/sensorZenithAngle" - source: variables/sensorZenithAngle - longName: "Sensor Zenith Angle" - units: "degree" - range: [0, 90] - - - name: "MetaData/sensorAzimuthAngle" - source: variables/sensorAzimuthAngle - longName: "Sensor Azimuth Angle" - units: "degree" - range: [0, 360] - - - name: "MetaData/sensorViewAngle" - source: variables/sensorScanAngle - longName: "Sensor View Angle" - units: "degree" - - - name: "MetaData/sensorChannelNumber" - source: variables/sensorChannelNumber - longName: "Sensor Channel Number" - - # ObsValue - - name: "ObsValue/brightnessTemperature" - coordinates: "longitude latitude Channel" - source: variables/brightnessTemperature - longName: "Brightness Temperature" - units: "K" - range: [100, 500] - chunks: [1000, 15] - compressionLevel: 4 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml b/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml index 345d4bd65..eff8c6f78 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml +++ b/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml @@ -85,7 +85,7 @@ observations: ioda: backend: netcdf - obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.amsua.$(splitvar)_ta.tm00.nc" + obsdataout: "{{ DATA }}/{{ RUN }}.t{{ cyc }}z.amsua.$(splitvar)_ta.tm00.nc" dimensions: - name: Channel diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml b/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml index d79c3a20e..f248a89db 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml +++ b/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml @@ -83,7 +83,7 @@ observations: ioda: backend: netcdf - obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.esamua.$(splitvar).tm00.nc" + obsdataout: "{{ DATA }}/{{ RUN }}.t{{ cyc }}z.esamua.$(splitvar).tm00.nc" dimensions: - name: Channel diff --git a/ush/ioda/bufr2ioda/bufr2ioda_base.py b/ush/ioda/bufr2ioda/bufr2ioda_base.py index 516560e1e..ef08b3503 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_base.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_base.py @@ -1,5 +1,6 @@ import json -import netCDF4 as nc +import os +import xarray as xr import re import yaml @@ -11,8 +12,9 @@ CPP = 1 PYTHON = 2 +GROUPS = ['MetaData', 'ObsValue'] -logger = Logger('BUFR2IODA_satwind_amv_goes.py', level='DEBUG') +logger = Logger(os.path.basename(__file__), level='DEBUG') class Bufr2IodaBase: @@ -28,7 +30,7 @@ def __init__(self, file_name, backend=PYTHON): self.splits = yaml_config['observations'][0]['obs space']['exports'].get('splits') self.sat_ids = None if self.splits: - self.sat_ids = self.splits['satId']['category']['map'].values() + self.sat_ids = [x for x in self.splits['satId']['category']['map'].values()] logger.info(self.splits) self.split_files = {} self.yaml_path = None @@ -51,32 +53,38 @@ def get_container(self): elif self.backend == CPP: container = {} for sat_id in self.sat_ids: + logger.info(f'Processing sat_id: {sat_id}') try: - container[sat_id] = nc.Dataset(self.split_files[sat_id], 'r+') + container[sat_id] = {} + file_name = self.split_files[sat_id] + container[sat_id]['dset'] = xr.open_dataset(file_name) + for group in GROUPS: + container[sat_id][group] = xr.open_dataset(file_name, group=group) except FileNotFoundError as e: - logger.info(e) + logger.info(f'File not existed exception for sat id: {sat_id} with error msg: {e}') + container.pop(sat_id) + self.sat_ids = container.keys() + logger.info(f'sat_ids included in the container: {container.keys()}') return container - def get_container_variable(self, container, variable, sat_id): + def get_container_variable(self, container, group, variable, sat_id): ret = None if self.backend == PYTHON: - ret = container.get(variable, sat_id) + ret = container.get(group + '/' + variable, sat_id) elif self.backend == CPP: - ret = container[sat_id][variable][:] + ret = container[sat_id][group][variable] return ret - def replace_container_variable(self, container, variable, var, sat_id): + def replace_container_variable(self, container, group, variable, var, sat_id): if self.backend == PYTHON: - container.replace(variable, sat_id) + container.replace(group + '/' + variable, sat_id) elif self.backend == CPP: - container[sat_id][variable][:] = var + container[sat_id][group][variable] = var def get_yaml_file(self): return self.config['yaml_file'] def get_yaml_config(self): - # TODO add and/or modify ymal contents from config json file. e.g. file names, etc. - # What does en_bufr2ioda_yaml.py do? connection to this? yaml_file = self.get_yaml_file() with open(yaml_file, 'r') as file: yaml_config = yaml.load(file, Loader=yaml.FullLoader) @@ -97,8 +105,13 @@ def ioda_encode(self, container): ioda_description = bufr.IodaDescription(self.yaml_path) bufr.IodaEncoder(ioda_description).encode(container) elif self.backend == CPP: - for item in container: - container[item].close() + for sat_id in container: + logger.info(f'Encode for {sat_id}') + container[sat_id]['dset'].close() + file_name = self.split_files[sat_id] + 'c' + container[sat_id]['dset'].to_netcdf(file_name) + for group in GROUPS: + container[sat_id][group].to_netcdf(file_name, mode='a', group=group) @timing_decorator def execute(self): diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index 95cf747a2..dec558bf6 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -7,14 +7,11 @@ from antcorr_application import ACCoeff, apply_ant_corr from utils import timing_decorator, nc_merge -logger = Logger('BUFR2IODA_satwind_amv_goes.py', level='DEBUG') +logger = Logger(os.path.basename(__file__), level='INFO') R1000 = 1000.0 R1000000 = 1000000.0 -SPLIT = ('metop-a', 'metop-b', 'metop-c', 'n17', 'n18', 'n19') -ioda_files = [(f'esamua.{x}.tm00.nc', f'amsua.{x}_ta.tm00.nc', f'amsua.{x}.tm00.nc') for x in SPLIT] - BACKEND = CPP @@ -36,11 +33,12 @@ def re_map_variable(self, container): # search the keyword “ta2tb” for details for sat_id in self.sat_ids: - ta = self.get_container_variable(container, 'ObsValue/brightnessTemperature', sat_id) + logger.info(f'Converting for {sat_id}, ...') + ta = self.get_container_variable(container, 'ObsValue', 'brightnessTemperature', sat_id) if ta.shape[0]: - ifov = self.get_container_variable(container, 'MetaData/sensorScanPosition', sat_id) + ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) tb = self.apply_ant_corr(sat_id, ta, ifov) - self.replace_container_variable(container, 'ObsValue/brightnessTemperature', tb, sat_id) + self.replace_container_variable(container, 'ObsValue', 'brightnessTemperature', tb, sat_id) def apply_ant_corr(self, sat_id, ta, ifov): ac = ACCoeff() # TODO add later @@ -50,7 +48,7 @@ def apply_ant_corr(self, sat_id, ta, ifov): # Convert antenna temperature to brightness temperature for i in range(ta.shape[0]): x = ta[i, :] - apply_ant_corr(ac, ifov[i], x) # TODO if this is too slow we might need to optimize it + apply_ant_corr(ac, int(ifov[i]), x) # TODO if this is too slow we might need to optimize it x[x > R1000] = R1000000 else: pass # TODO after know how to set llll @@ -58,29 +56,35 @@ def apply_ant_corr(self, sat_id, ta, ifov): @timing_decorator -def merge(amsua_files): - amsua_0 = amsua_files[0] - amsua_1 = amsua_files[1] - obs_Path_prefix = os.path.commonprefix([amsua_files[0][0], amsua_files[1][0]]) +def merge(amsua_files, splits): + ioda_files = [(f'esamua.{x}.tm00.ncc', f'amsua.{x}_ta.tm00.ncc', f'amsua.{x}.tm00.nc') for x in splits] + logger.info(f'Ioda files: {ioda_files}') + file1 = [f for f in amsua_files[0].values()] + file2 = [f for f in amsua_files[1].values()] + obs_path_prefix = os.path.commonprefix([file1[0], file2[0]]) + logger.info(f'common prefix: {obs_path_prefix}') for ioda_file_0, ioda_file_1, ioda_file_t in ioda_files: try: - nc_merge(obs_Path_prefix + ioda_file_0, obs_Path_prefix + ioda_file_1, obs_Path_prefix + ioda_file_t) + nc_merge(obs_path_prefix + ioda_file_0, + obs_path_prefix + ioda_file_1, + obs_path_prefix + ioda_file_t + ) except FileNotFoundError as e: - print(e) + logger.info(f'File not found exception: {e}') if __name__ == '__main__': parser = argparse.ArgumentParser() - # parser.add_argument('-t', '--type', type=str, help='Input Satellite type a/e', required=True) parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) parser.add_argument('-v', '--verbose', help='print debug logging information', action='store_true') args = parser.parse_args() log_level = 'DEBUG' if args.verbose else 'INFO' - logger = Logger('BUFR2IODA_ncep_amsua.py', level=log_level) + logger = Logger(os.path.basename(__file__), level=log_level) amsua_files = [] + splits = set() for sat_type in ['a', 'e']: - print(sat_type) + logger.info(f'Processing sat type: {sat_type}') if sat_type == 'a': convert = Bufr2IodaAmusa(args.config, backend=BACKEND) else: @@ -88,9 +92,9 @@ def merge(amsua_files): convert.execute() amsua_files.append(convert.split_files) - print(amsua_files) logger.info('Converting amsua type {} done!'.format(sat_type)) + splits.update(set(convert.sat_ids)) - print('--start to merge--') - merge(amsua_files) - print('--Finished merge--') + logger.info('--start to merge--') + merge(amsua_files, splits) + logger.info('--Finished merge--') diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 066642ab9..5b03f3427 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -53,7 +53,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # copy necessary fix files to runtime directory fix_files = ["atms_beamwidth.txt", - "amsua_metop-c_v2.ACCoeff.nc" + "amsua_metop-c_v2.ACCoeff.nc" # TODO need to find a place for this .nc file. ] for fix_file in fix_files: shutil.copy(os.path.join(config_template_dir, fix_file), os.path.join(DATA, fix_file)) diff --git a/ush/ioda/bufr2ioda/utils.py b/ush/ioda/bufr2ioda/utils.py index b1f7c62e7..76c3b7ba0 100644 --- a/ush/ioda/bufr2ioda/utils.py +++ b/ush/ioda/bufr2ioda/utils.py @@ -1,6 +1,9 @@ import netCDF4 as nc import numpy as np +import os import time +from wxflow import Logger +logger = Logger(os.path.basename(__file__), level='INFO') def nc_merge(file_name1, file_name2, target_name): @@ -21,7 +24,7 @@ def nc_merge(file_name1, file_name2, target_name): # Create groups for grp in ncf1.groups: - print(grp) + logger.info(grp) ncf.createGroup(grp) # Create dimensions @@ -37,7 +40,7 @@ def nc_merge(file_name1, file_name2, target_name): dim_size = dim1.size ncf.createDimension(dim1.name, dim_size) else: - print('error: dimension mismatch') # TODO throw an exception + logger.info('error: dimension mismatch') # TODO throw an exception # Create Dimension variables vars2 = ncf2.variables @@ -63,7 +66,7 @@ def nc_merge(file_name1, file_name2, target_name): if vars2.get(key): var2 = vars2[key] else: - print(f"-----------Warning: {key} is not existed in ncf2, skip -----------------") + logger.info(f"-----------Warning: {key} is not existed in ncf2, skip -----------------") continue var_array = np.concatenate((var, var2)) var_path = f"{grp1[k].path}/{var.name}" @@ -82,6 +85,6 @@ def wrapper(*args, **kwargs): result = func(*args, **kwargs) end_time = time.time() execution_time = end_time - start_time - print(f"Function '{func.__name__}' took {execution_time:.6f} seconds to execute.") + logger.info(f"Function '{func.__name__}' took {execution_time:.6f} seconds to execute.") return result return wrapper From 812904cbc9713db34345fb70a389339628a7c383 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Wed, 29 Nov 2023 09:33:22 -0600 Subject: [PATCH 43/58] revert CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6bed0bba3..b26b64b5b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,7 +107,7 @@ if(BUILD_GDASBUNDLE) # Build IODA converters option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON) if(BUILD_IODA_CONVERTERS) - ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" BRANCH develop) + ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" BRANCH develop ) endif() # Land associated repositories From deddffdf18aacfa3fe104c651cf0d22549294775 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Wed, 29 Nov 2023 10:05:44 -0600 Subject: [PATCH 44/58] revert parm/atm/variational/3dvar_dripcg.yaml --- parm/atm/variational/3dvar_dripcg.yaml | 36 +++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml index 2f9ccb76f..abd945842 100644 --- a/parm/atm/variational/3dvar_dripcg.yaml +++ b/parm/atm/variational/3dvar_dripcg.yaml @@ -33,7 +33,7 @@ variational: minimizer: algorithm: DRIPCG iterations: - - ninner: 1 + - ninner: 2 gradient norm reduction: 1e-10 test: on geometry: @@ -50,6 +50,40 @@ variational: field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml diagnostics: departures: bkgmob + - ninner: 4 + gradient norm reduction: 1e-10 + test: on + geometry: + fms initialization: + namelist filename: ./fv3jedi/fmsmpp.nml + field table filename: ./fv3jedi/field_table + akbk: ./fv3jedi/akbk.nc4 + layout: + - $(layout_x) + - $(layout_y) + npx: $(npx_anl) + npy: $(npy_anl) + npz: $(npz_anl) + field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml + diagnostics: + departures: bkgmob1 final: diagnostics: departures: anlmob + increment: + output: + filetype: auxgrid + gridtype: gaussian + filename: ./anl/atminc. + geometry: + fms initialization: + namelist filename: ./fv3jedi/fmsmpp.nml + field table filename: ./fv3jedi/field_table + akbk: ./fv3jedi/akbk.nc4 + layout: + - $(layout_x) + - $(layout_y) + npx: $(npx_anl) + npy: $(npy_anl) + npz: $(npz_anl) + field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml From f29a762af079fd9755e5c00642f232f57a01b59f Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Thu, 30 Nov 2023 12:06:34 -0600 Subject: [PATCH 45/58] for test --- .../bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml | 2 +- .../ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml | 2 +- .../bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 10 ++++----- .../{bufr2ioda_base.py => combine_base.py} | 0 ush/ioda/bufr2ioda/run_bufr2ioda.py | 21 ++++++++++++------- 5 files changed, 21 insertions(+), 14 deletions(-) rename ush/ioda/bufr2ioda/{bufr2ioda_base.py => combine_base.py} (100%) diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml b/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml index eff8c6f78..345d4bd65 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml +++ b/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml @@ -85,7 +85,7 @@ observations: ioda: backend: netcdf - obsdataout: "{{ DATA }}/{{ RUN }}.t{{ cyc }}z.amsua.$(splitvar)_ta.tm00.nc" + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.amsua.$(splitvar)_ta.tm00.nc" dimensions: - name: Channel diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml b/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml index f248a89db..7d6d377a4 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml +++ b/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml @@ -83,7 +83,7 @@ observations: ioda: backend: netcdf - obsdataout: "{{ DATA }}/{{ RUN }}.t{{ cyc }}z.esamua.$(splitvar).tm00.nc" + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.esamua.$(splitvar).tm00.nc" dimensions: - name: Channel diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index dec558bf6..9ef113d4e 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -2,7 +2,7 @@ import argparse import os -from bufr2ioda_base import Bufr2IodaBase, CPP +from combine_base import Bufr2IodaBase, CPP from wxflow import Logger from antcorr_application import ACCoeff, apply_ant_corr from utils import timing_decorator, nc_merge @@ -35,10 +35,10 @@ def re_map_variable(self, container): for sat_id in self.sat_ids: logger.info(f'Converting for {sat_id}, ...') ta = self.get_container_variable(container, 'ObsValue', 'brightnessTemperature', sat_id) - if ta.shape[0]: - ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) - tb = self.apply_ant_corr(sat_id, ta, ifov) - self.replace_container_variable(container, 'ObsValue', 'brightnessTemperature', tb, sat_id) + # if ta.shape[0]: + # ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) + # tb = self.apply_ant_corr(sat_id, ta, ifov) + # self.replace_container_variable(container, 'ObsValue', 'brightnessTemperature', tb, sat_id) def apply_ant_corr(self, sat_id, ta, ifov): ac = ACCoeff() # TODO add later diff --git a/ush/ioda/bufr2ioda/bufr2ioda_base.py b/ush/ioda/bufr2ioda/combine_base.py similarity index 100% rename from ush/ioda/bufr2ioda/bufr2ioda_base.py rename to ush/ioda/bufr2ioda/combine_base.py diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 5b03f3427..e7a0f1386 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -58,10 +58,17 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): for fix_file in fix_files: shutil.copy(os.path.join(config_template_dir, fix_file), os.path.join(DATA, fix_file)) + # get all observation types to be processed by python + BUFR_total_py_files = set(glob.glob(os.path.join(USH_IODA, 'bufr2ioda_*.py'))) + combine_type = 'bufr2ioda_combine_' + BUFR_combine_files = set(glob.glob(os.path.join(USH_IODA, combine_type + '*.py'))) + # Specify observation types to be processed by a script - BUFR_py_files = glob.glob(os.path.join(USH_IODA, 'bufr2ioda_*.py')) + BUFR_py_files = BUFR_total_py_files.difference(BUFR_combine_files) + BUFR_py_files = [os.path.basename(f) for f in BUFR_py_files] BUFR_py = [f.replace('bufr2ioda_', '').replace('.py', '') for f in BUFR_py_files] + logger.info(f'All obs type processed by python: {BUFR_py}') config_files = [] exename = [] @@ -87,6 +94,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): BUFR_yaml_files = glob.glob(os.path.join(config_template_dir, '*.yaml')) BUFR_yaml_files = [os.path.basename(f) for f in BUFR_yaml_files] BUFR_yaml = [f.replace('bufr2ioda_', '').replace('.yaml', '') for f in BUFR_yaml_files] + logger.info(f'All obs type processed by yaml: {BUFR_yaml}') for obtype in BUFR_yaml: logger.info(f"Convert {obtype}...") @@ -99,8 +107,8 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): bufr2iodaexe = BIN_GDAS + '/bufr2ioda.x' # append the values to the lists - config_files.append(yaml_output_file) - exename.append(bufr2iodaexe) + #config_files.append(yaml_output_file) + #exename.append(bufr2iodaexe) # Check if the converter was successful # if os.path.exists(yaml_output_file): @@ -113,10 +121,9 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): config_files = [] exename = [] # Specify observation types to be processed by a script with combined methods - script_type = 'bufr2ioda_combine_' - BUFR_combine_files = glob.glob(os.path.join(USH_IODA, script_type + '*.py')) BUFR_combine_files = [os.path.basename(f) for f in BUFR_combine_files] - BUFR_combine = [f.replace(script_type, '').replace('.py', '') for f in BUFR_combine_files] + BUFR_combine = [f.replace(combine_type, '').replace('.py', '') for f in BUFR_combine_files] + logger.info(f'All obs type processed by combine: {BUFR_combine}') for obtype in BUFR_combine: logger.info(f"Convert {obtype}...") @@ -126,7 +133,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): gen_bufr_json(config, template, json_output_file) # Use the converter script for the ob type - bufr2iodapy = os.path.join(USH_IODA, script_type + obtype + ".py") + bufr2iodapy = os.path.join(USH_IODA, combine_type + obtype + ".py") # append the values to the lists config_files.append(json_output_file) From ce8fad83de699d9f4525c5cd4110d53c5f2f8760 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Thu, 30 Nov 2023 14:46:25 -0600 Subject: [PATCH 46/58] a few small changes --- parm/ioda/bufr2ioda/bufr2ioda_ncep_amsua.json | 6 ++++-- ush/ioda/bufr2ioda/antcorr_application.py | 5 +++-- ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 13 ++++++++----- ush/ioda/bufr2ioda/run_bufr2ioda.py | 5 +++-- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ncep_amsua.json b/parm/ioda/bufr2ioda/bufr2ioda_ncep_amsua.json index 5f6153b42..cdde43797 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_ncep_amsua.json +++ b/parm/ioda/bufr2ioda/bufr2ioda_ncep_amsua.json @@ -14,6 +14,8 @@ { "satellite_name": "GOES-17", "satellite_full_name": "Geostationary Operational Satellite - 17", "satellite_id": 271 }, { "satellite_name": "GOES-18", "satellite_full_name": "Geostationary Operational Satellite - 18", "satellite_id": 272 } ], - "yaml_file" : ["{{ COM_OBS }}/ncep_esamua_{{ current_cycle | to_YMDH }}.yaml", - "{{ COM_OBS }}/ncep_1bamua_ta_{{ current_cycle | to_YMDH }}.yaml"] + "yaml_file" : ["{{ DATA }}/ncep_esamua_{{ current_cycle | to_YMDH }}.yaml", + "{{ DATA }}/ncep_1bamua_ta_{{ current_cycle | to_YMDH }}.yaml"], + "ac_dir" : "{{ DATA }}" + } diff --git a/ush/ioda/bufr2ioda/antcorr_application.py b/ush/ioda/bufr2ioda/antcorr_application.py index fb541c96a..e8b63301b 100644 --- a/ush/ioda/bufr2ioda/antcorr_application.py +++ b/ush/ioda/bufr2ioda/antcorr_application.py @@ -1,4 +1,5 @@ import netCDF4 as nc +import os import numpy as np INVALID = -1.0 @@ -8,8 +9,8 @@ class ACCoeff: - def __init__(self, sat_id='metop-c'): - file_name = 'amsua_' + sat_id + '_v2.ACCoeff.nc' + def __init__(self, ac_dir, sat_id='metop-c'): + file_name = os.path.join(ac_dir, 'amsua_' + sat_id + '_v2.ACCoeff.nc') nc_file = nc.Dataset(file_name) self.n_FOVS = len(nc_file.dimensions['n_FOVs']) self.n_Channels = len(nc_file.dimensions['n_Channels']) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index 9ef113d4e..ca698792c 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -25,6 +25,9 @@ class Bufr2IodaEbmua(Bufr2IodaBase): def get_yaml_file(self): return self.config['yaml_file'][1] + def get_ac_dir(self): + return self.config['ac_dir'] + @timing_decorator def re_map_variable(self, container): # TODO replace this follow that in GSI @@ -35,13 +38,13 @@ def re_map_variable(self, container): for sat_id in self.sat_ids: logger.info(f'Converting for {sat_id}, ...') ta = self.get_container_variable(container, 'ObsValue', 'brightnessTemperature', sat_id) - # if ta.shape[0]: - # ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) - # tb = self.apply_ant_corr(sat_id, ta, ifov) - # self.replace_container_variable(container, 'ObsValue', 'brightnessTemperature', tb, sat_id) + if ta.shape[0]: + ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) + tb = self.apply_ant_corr(sat_id, ta, ifov) + self.replace_container_variable(container, 'ObsValue', 'brightnessTemperature', tb, sat_id) def apply_ant_corr(self, sat_id, ta, ifov): - ac = ACCoeff() # TODO add later + ac = ACCoeff(self.get_ac_dir()) # TODO add later llll = 1 # TODO how to set this if llll == 1: if sat_id not in ['n15', 'n16']: diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index e7a0f1386..1ae7db797 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -45,6 +45,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): config = { 'RUN': RUN, 'current_cycle': current_cycle, + 'DATA': DATA, 'DMPDIR': DMPDIR, 'COM_OBS': COM_OBS, 'PDY': current_cycle.strftime('%Y%m%d'), @@ -107,8 +108,8 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): bufr2iodaexe = BIN_GDAS + '/bufr2ioda.x' # append the values to the lists - #config_files.append(yaml_output_file) - #exename.append(bufr2iodaexe) + config_files.append(yaml_output_file) + exename.append(bufr2iodaexe) # Check if the converter was successful # if os.path.exists(yaml_output_file): From 941cb2b1915276a5eb28e2f5f330169df380cd28 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Fri, 1 Dec 2023 08:20:56 -0600 Subject: [PATCH 47/58] optimized apply_ant_corr --- ush/ioda/bufr2ioda/antcorr_application.py | 35 +++++++++---------- .../bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 7 ++-- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/ush/ioda/bufr2ioda/antcorr_application.py b/ush/ioda/bufr2ioda/antcorr_application.py index e8b63301b..fec9a1430 100644 --- a/ush/ioda/bufr2ioda/antcorr_application.py +++ b/ush/ioda/bufr2ioda/antcorr_application.py @@ -12,29 +12,26 @@ class ACCoeff: def __init__(self, ac_dir, sat_id='metop-c'): file_name = os.path.join(ac_dir, 'amsua_' + sat_id + '_v2.ACCoeff.nc') nc_file = nc.Dataset(file_name) - self.n_FOVS = len(nc_file.dimensions['n_FOVs']) - self.n_Channels = len(nc_file.dimensions['n_Channels']) - self.A_earth = nc_file.variables['A_earth'][:] - self.A_platform = nc_file.variables['A_platform'][:] - self.A_space = nc_file.variables['A_space'][:] + self.n_fovs = len(nc_file.dimensions['n_FOVs']) + self.n_channels = len(nc_file.dimensions['n_Channels']) + self.a_earth = nc_file.variables['A_earth'][:] + self.a_platform = nc_file.variables['A_platform'][:] + self.a_space = nc_file.variables['A_space'][:] + self.a_ep = self.a_earth + self.a_platform + self.a_sp = self.a_space * TSPACE -def remove_ant_corr(AC, iFOV, T): +def remove_ant_corr(i, ac, ifov, t): # AC: Structure containing the antenna correction coefficients for the sensor of interest. # iFOV: The FOV index for a scanline of the sensor of interest. # T: On input, this argument contains the brightness + t = ac.a_ep[i, ifov] * t + ac.a_sp[i, ifov] + t[(ifov < 1) | (ifov > ac.n_fovs)] = [INVALID] + return t - if iFOV < 1 or iFOV > AC.n_FOVS or len(T) != AC.n_Channels: - return [INVALID] * len(T) - iFOV = iFOV - 1 - T = AC.A_earth[:, iFOV] * T + AC.A_platform[:, iFOV] * T + AC.A_space[:, iFOV] * TSPACE - return T - -def apply_ant_corr(AC, iFOV, T): - # T: On input, this argument contains the antenna temperatures for the sensor channels. - if iFOV < 1 or iFOV > AC.n_FOVS or len(T) != AC.n_Channels: - return [INVALID] * len(T) - iFOV = iFOV - 1 - T = (T - AC.A_space[:, iFOV] * TSPACE) / (AC.A_earth[:, iFOV] + AC.A_platform[:, iFOV]) - return T +def apply_ant_corr(i, ac, ifov, t): + # t: on input, this argument contains the antenna temperatures for the sensor channels. + t = (t - ac.a_sp[i, ifov]) / ac.a_ep[i, ifov] + t[(ifov < 1) | (ifov > ac.n_fovs)] = [INVALID] + return t diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index ca698792c..2b4b276b5 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -49,9 +49,10 @@ def apply_ant_corr(self, sat_id, ta, ifov): if llll == 1: if sat_id not in ['n15', 'n16']: # Convert antenna temperature to brightness temperature - for i in range(ta.shape[0]): - x = ta[i, :] - apply_ant_corr(ac, int(ifov[i]), x) # TODO if this is too slow we might need to optimize it + ifov = ifov.astype(int) - 1 + for i in range(ta.shape[1]): + x = ta[:, i] + apply_ant_corr(i, ac, ifov, x) x[x > R1000] = R1000000 else: pass # TODO after know how to set llll From 654dcdf11f1ecb31cbb519fc2c90f2e8f4c172cf Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:13:43 -0500 Subject: [PATCH 48/58] Minor observation YAML updates for ATMS, SATWIND and SCATWIND (#789) This PR includes the following: 1. YAML name change - add sensor (ABI) to the filename of observation YAML for SATWIND from GOES platforms 2. YAML name change - add sensor (ASCAT) to the filename of observation YAML for SCATWIND from MetOp platforms 3. update the YAML name changes in the list for 3dvar 4. Add `CO2` in obs forward operator and add `linear obs operator` without CO2 for ATMS These changes have no impact on 3DVar results. --- parm/atm/obs/config/atms_n20.yaml | 4 +++- parm/atm/obs/config/atms_npp.yaml | 4 +++- .../{satwind_goes-16.yaml => satwind_abi_goes-16.yaml} | 0 .../{satwind_goes-17.yaml => satwind_abi_goes-17.yaml} | 0 ...atwind_metop-a.yaml => scatwind_ascat_metop-a.yaml} | 0 ...atwind_metop-b.yaml => scatwind_ascat_metop-b.yaml} | 0 parm/atm/obs/lists/gdas_prototype_3d.yaml | 10 +++++----- 7 files changed, 11 insertions(+), 7 deletions(-) rename parm/atm/obs/config/{satwind_goes-16.yaml => satwind_abi_goes-16.yaml} (100%) rename parm/atm/obs/config/{satwind_goes-17.yaml => satwind_abi_goes-17.yaml} (100%) rename parm/atm/obs/config/{scatwind_metop-a.yaml => scatwind_ascat_metop-a.yaml} (100%) rename parm/atm/obs/config/{scatwind_metop-b.yaml => scatwind_ascat_metop-b.yaml} (100%) diff --git a/parm/atm/obs/config/atms_n20.yaml b/parm/atm/obs/config/atms_n20.yaml index 2a29f5d95..5cb9c27f3 100644 --- a/parm/atm/obs/config/atms_n20.yaml +++ b/parm/atm/obs/config/atms_n20.yaml @@ -15,7 +15,7 @@ obs space: obs operator: name: CRTM - Absorbers: [H2O,O3] + Absorbers: [H2O, O3, CO2] Clouds: [Water, Ice] Cloud_Fraction: 1.0 Cloud_Seeding: true @@ -23,6 +23,8 @@ obs operator: Sensor_ID: &Sensor_ID atms_n20 EndianType: little_endian CoefficientPath: $(DATA)/crtm/ + linear obs operator: + Absorbers: [H2O, O3] obs bias: input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias.nc4 diff --git a/parm/atm/obs/config/atms_npp.yaml b/parm/atm/obs/config/atms_npp.yaml index ecddd346c..0a9b633b6 100644 --- a/parm/atm/obs/config/atms_npp.yaml +++ b/parm/atm/obs/config/atms_npp.yaml @@ -15,7 +15,7 @@ obs space: obs operator: name: CRTM - Absorbers: [H2O,O3] + Absorbers: [H2O,O3,CO2] Clouds: [Water, Ice] Cloud_Fraction: 1.0 Cloud_Seeding: true @@ -23,6 +23,8 @@ obs operator: Sensor_ID: &Sensor_ID atms_npp EndianType: little_endian CoefficientPath: $(DATA)/crtm/ + linear obs operator: + Absorbers: [H2O, O3] obs bias: input file: $(DATA)/obs/$(GPREFIX)atms_npp.satbias.nc4 diff --git a/parm/atm/obs/config/satwind_goes-16.yaml b/parm/atm/obs/config/satwind_abi_goes-16.yaml similarity index 100% rename from parm/atm/obs/config/satwind_goes-16.yaml rename to parm/atm/obs/config/satwind_abi_goes-16.yaml diff --git a/parm/atm/obs/config/satwind_goes-17.yaml b/parm/atm/obs/config/satwind_abi_goes-17.yaml similarity index 100% rename from parm/atm/obs/config/satwind_goes-17.yaml rename to parm/atm/obs/config/satwind_abi_goes-17.yaml diff --git a/parm/atm/obs/config/scatwind_metop-a.yaml b/parm/atm/obs/config/scatwind_ascat_metop-a.yaml similarity index 100% rename from parm/atm/obs/config/scatwind_metop-a.yaml rename to parm/atm/obs/config/scatwind_ascat_metop-a.yaml diff --git a/parm/atm/obs/config/scatwind_metop-b.yaml b/parm/atm/obs/config/scatwind_ascat_metop-b.yaml similarity index 100% rename from parm/atm/obs/config/scatwind_metop-b.yaml rename to parm/atm/obs/config/scatwind_ascat_metop-b.yaml diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml b/parm/atm/obs/lists/gdas_prototype_3d.yaml index e8f2d006c..1c4d3a95c 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml @@ -4,10 +4,10 @@ observers: ##- !INC ${OBS_YAML_DIR}/atms_n20.yaml ##- !INC ${OBS_YAML_DIR}/atms_npp.yaml ##- !INC ${OBS_YAML_DIR}/aircraft.yaml -- !INC ${OBS_YAML_DIR}/satwind_goes-16.yaml -##- !INC ${OBS_YAML_DIR}/satwind_goes-17.yaml -##- !INC ${OBS_YAML_DIR}/scatwind_metop-a.yaml -##- !INC ${OBS_YAML_DIR}/scatwind_metop-b.yaml +##- !INC ${OBS_YAML_DIR}/satwind_abi_goes-16.yaml +##- !INC ${OBS_YAML_DIR}/satwind_abi_goes-17.yaml +##- !INC ${OBS_YAML_DIR}/scatwind_ascat_metop-a.yaml +##- !INC ${OBS_YAML_DIR}/scatwind_ascat_metop-b.yaml ##- !INC ${OBS_YAML_DIR}/omi_aura.yaml ##- !INC ${OBS_YAML_DIR}/ompsnp_npp.yaml ##- !INC ${OBS_YAML_DIR}/ompstc8_npp.yaml @@ -15,4 +15,4 @@ observers: ##- !INC ${OBS_YAML_DIR}/cris-fsr_npp.yaml ##- !INC ${OBS_YAML_DIR}/sfc.yaml ##- !INC ${OBS_YAML_DIR}/sfcship.yaml -##- !INC ${OBS_YAML_DIR}/gnssro.yaml \ No newline at end of file +##- !INC ${OBS_YAML_DIR}/gnssro.yaml From de8ff12b797317537e02ea56ee14262b3b0e6f69 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Mon, 4 Dec 2023 20:47:19 -0600 Subject: [PATCH 49/58] add remove function --- .../bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 56 +++++++++++++++---- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index 2b4b276b5..a6d4afe98 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 import argparse +import json import os from combine_base import Bufr2IodaBase, CPP from wxflow import Logger -from antcorr_application import ACCoeff, apply_ant_corr +from antcorr_application import ACCoeff, apply_ant_corr, remove_ant_corr from utils import timing_decorator, nc_merge logger = Logger(os.path.basename(__file__), level='INFO') @@ -14,16 +15,31 @@ BACKEND = CPP +AMSUA_TYPE_CHANGE_DATETIME = "2022120000" + +YAML_NORMAL = True # current as normal + class Bufr2IodaAmusa(Bufr2IodaBase): + + def __init__(self, yaml_order, *args, **kwargs): + self.yaml_order = yaml_order + super().__init__(*args, **kwargs) + def get_yaml_file(self): - return self.config['yaml_file'][0] + if self.yaml_order: + return self.config['yaml_file'][0] + else: + return self.config['yaml_file'][1] -class Bufr2IodaEbmua(Bufr2IodaBase): +class Bufr2IodaAmusaChange(Bufr2IodaAmusa): def get_yaml_file(self): - return self.config['yaml_file'][1] + if self.yaml_order: + return self.config['yaml_file'][1] + else: + return self.config['yaml_file'][0] def get_ac_dir(self): return self.config['ac_dir'] @@ -39,11 +55,14 @@ def re_map_variable(self, container): logger.info(f'Converting for {sat_id}, ...') ta = self.get_container_variable(container, 'ObsValue', 'brightnessTemperature', sat_id) if ta.shape[0]: - ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) - tb = self.apply_ant_corr(sat_id, ta, ifov) - self.replace_container_variable(container, 'ObsValue', 'brightnessTemperature', tb, sat_id) - - def apply_ant_corr(self, sat_id, ta, ifov): + if self.yaml_order: + ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) + else: + ifov = self.get_container_variable(container, 'MetaData', 'fieldOfViewNumber', sat_id) + tb = self.apply_corr(sat_id, ta, ifov) + self.replace_container_variable(container, 'ObsValue', 'brightnessTemperature', tb, sat_id) + + def apply_corr(self, sat_id, ta, ifov): ac = ACCoeff(self.get_ac_dir()) # TODO add later llll = 1 # TODO how to set this if llll == 1: @@ -52,7 +71,10 @@ def apply_ant_corr(self, sat_id, ta, ifov): ifov = ifov.astype(int) - 1 for i in range(ta.shape[1]): x = ta[:, i] - apply_ant_corr(i, ac, ifov, x) + if self.yaml_order: + apply_ant_corr(i, ac, ifov, x) + else: + remove_ant_corr(i, ac, ifov, x) x[x > R1000] = R1000000 else: pass # TODO after know how to set llll @@ -87,12 +109,22 @@ def merge(amsua_files, splits): logger = Logger(os.path.basename(__file__), level=log_level) amsua_files = [] splits = set() + json_file_name = args.config + with open(json_file_name, "r") as json_file: + config = json.load(json_file) + + cycle_datetime = config["cycle_datetime"] + if cycle_datetime >= AMSUA_TYPE_CHANGE_DATETIME: + yaml_order = YAML_NORMAL + else: + yaml_order = not YAML_NORMAL + for sat_type in ['a', 'e']: logger.info(f'Processing sat type: {sat_type}') if sat_type == 'a': - convert = Bufr2IodaAmusa(args.config, backend=BACKEND) + convert = Bufr2IodaAmusa(yaml_order, args.config, backend=BACKEND) else: - convert = Bufr2IodaEbmua(args.config, backend=BACKEND) + convert = Bufr2IodaAmusaChange(yaml_order, args.config, backend=BACKEND) convert.execute() amsua_files.append(convert.split_files) From 30ab4f928846b5a97235367fb76f51be038abf07 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Tue, 5 Dec 2023 11:48:02 -0500 Subject: [PATCH 50/58] Update IASI YAML files for end-to-end testing (#769) --- parm/atm/obs/config/iasi_metop-a.yaml | 756 ++++++++++++++++++++ parm/atm/obs/config/iasi_metop-b.yaml | 756 ++++++++++++++++++++ parm/atm/obs/lists/gdas_prototype_3d.yaml | 2 + parm/atm/obs/testing/iasi_metop-a.yaml | 22 +- parm/atm/obs/testing/iasi_metop-a_noqc.yaml | 158 ++++ parm/atm/obs/testing/iasi_metop-b.yaml | 22 +- parm/atm/obs/testing/iasi_metop-b_noqc.yaml | 158 ++++ parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml | 238 ++++++ 8 files changed, 2098 insertions(+), 14 deletions(-) create mode 100644 parm/atm/obs/config/iasi_metop-a.yaml create mode 100644 parm/atm/obs/config/iasi_metop-b.yaml create mode 100644 parm/atm/obs/testing/iasi_metop-a_noqc.yaml create mode 100644 parm/atm/obs/testing/iasi_metop-b_noqc.yaml create mode 100755 parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml diff --git a/parm/atm/obs/config/iasi_metop-a.yaml b/parm/atm/obs/config/iasi_metop-a.yaml new file mode 100644 index 000000000..f5c7a602a --- /dev/null +++ b/parm/atm/obs/config/iasi_metop-a.yaml @@ -0,0 +1,756 @@ +obs space: + name: iasi_metop-a + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)mtiasi_metop-a.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_iasi_metop-a_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + observed variables: [radiance] + simulated variables: [brightnessTemperature] + derived variables: [brightnessTemperature] + channels: &iasi_metop-a_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 + +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: iasi_metop-a + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + linear obs operator: + Absorbers: [H2O,O3] + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)iasi_metop-a.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)iasi_metop-a.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-a_tlapse $(DATA)/obs/$(GPREFIX)iasi_metop-a.tlapse.txt + - name: lapse_rate + tlapse: *iasi_metop-a_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)iasi_metop-a.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)iasi_metop-a.satbias_cov.nc4 + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +#- filter: Create Diagnostic Flags +# filter variables: +# - name: brightnessTemperature +# channels: *iasi_metop-a_channels +# flags: +# - name: ScanEdgeRemoval +# initial value: false +# force reinitialization: false +# - name: Thinning +# initial value: false +# force reinitialization: false +# - name: ShortwaveIRCheck +# initial value: false +# force reinitialization: false +# - name: ObsValueRangeCheck +# initial value: false +# force reinitialization: false +# - name: CloudDetection +# initial value: false +# force reinitialization: false +# - name: NSSTCheck +# initial value: false +# force reinitialization: false +# - name: GrossCheck +# initial value: false +# force reinitialization: false +# - name: InterChannelConsistency +# initial value: false +# force reinitialization: false +# - name: UseFlagCheck +# initial value: false +# force reinitialization: false + +# Step 0-B: Create Derived Variables +# Assign channel wavenumbers in m-1 +- filter: Variable Assignment + assignments: + - name: MetaData/sensorCentralWavenumber + type: float + channels: *iasi_metop-a_channels + function: + name: ObsFunction/LinearCombination + options: + variables: + - name: ObsValue/radiance + channels: *iasi_metop-a_channels + coefs: [25.0] + intercept: 64475.0 + use channel numbers: true + +# Transform radiance to brightness temperature +- filter: Variable Transforms + Transform: SatBrightnessTempFromRad + transform from: + name: ObsValue/radiance + channels: *iasi_metop-a_channels + spectral variable: + name: MetaData/sensorCentralWavenumber + channels: *iasi_metop-a_channels + radiance units: wavenumber + planck1: 1.191042953e-16 + planck2: 1.4387774e-2 + +# Step 0-C: Assign Observation Error +obs prior filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: assign error + error parameter vector: &iasi_metop-a_oberr [ 1.38, 0.81, 0.75, 0.79, 0.72, 0.74, 0.68, 0.72, 0.65, 0.65, + 0.65, 0.69, 0.64, 0.64, 0.65, 0.67, 0.62, 0.61, 0.62, 0.64, 0.59, 0.76, + 1.22, 0.78, 0.64, 0.62, 0.61, 0.69, 0.65, 0.59, 0.61, 0.59, 0.68, 0.62, + 0.68, 4.38, 3.05, 2.31, 1.56, 1.33, 1.58, 0.93, 1.67, 0.72, 0.57, 0.58, + 0.55, 0.68, 0.59, 0.68, 0.59, 0.65, 0.58, 0.62, 0.64, 0.58, 0.64, 0.55, + 0.64, 0.5, 0.82, 0.59, 0.62, 0.51, 0.64, 0.52, 0.51, 0.51, 0.76, 0.52, + 0.57, 0.55, 0.69, 0.58, 0.65, 0.61, 0.59, 0.64, 0.76, 0.72, 1.05, 0.75, + 0.51, 0.65, 1.3, 0.69, 0.93, 1.49, 1.12, 0.68, 0.66, 0.67, 0.59, 0.59, + 0.69, 0.67, 0.64, 0.62, 0.72, 0.69, 0.66, 0.79, 0.78, 0.74, 0.88, 0.77, + 0.88, 0.86, 1, 0.87, 0.85, 0.88, 0.84, 0.84, 0.84, 0.8, 0.8, 0.87, + 0.98, 0.52, 0.65, 0.69, 0.61, 0.6, 0.67, 0.79, 0.62, 0.66, 0.7, 0.65, + 0.62, 0.61, 0.62, 0.53, 0.6, 0.68, 0.95, 0.63, 0.97, 0.65, 0.98, 0.58, + 0.73, 0.65, 0.85, 0.99, 0.76, 0.85, 0.97, 0.77, 0.62, 0.63, 1.21, 1.41, + 1.55, 1.78, 1.35, 1.14, 1.69, 1.79, 1.46, 1.63, 1.94, 2.01, 1.24, 1.76, + 1.26, 1.47, 1.9, 1.66, 2.13, 1.49, 1.52, 1.55, 1.96, 2.31, 2.33, 2.32, + 2.31, 2.33, 2.23, 2.33, 1.84, 2.29, 2.28, 2.28, 2.28, 2.26, 2.26, 2.26, + 2.27, 2.24, 2.23, 2.24, 2.26, 2.28, 2.28, 2.3, 2.15, 2.31, 2.37, 2.27, + 2.29, 2.29, 2.23, 2.28, 2.32, 2.32, 2.31, 2.32, 2.32, 2.31, 2.31, 2.28, + 2.29, 2.28, 2.26, 2.29, 2.27, 2.26, 2.25, 2.27, 2.24, 2.21, 2.24, 2.17, + 2.18, 2.17, 2.21, 1.99, 2.16, 2.2, 2.13, 2.12, 2.13, 2.1, 2.12, 2.11, + 2.09, 2.09, 2.08, 2.09, 2.04, 2.04, 2.1, 2.01, 2.05, 2.03, 2.06, 1.98, + 1.95, 1.94, 1.91, 1.7, 1.76, 1.77, 1.83, 2.04, 1.91, 1.99, 1.99, 2.07, + 2.02, 2.04, 2.1, 2.06, 2.18, 2.21, 2.24, 2.23, 2.23, 1.98, 2.2, 2.18, + 2.18, 2.21, 2.23, 2.24, 2.24, 2.25, 1.8, 2.24, 1.73, 1.73, 2.27, 1.67, + 2.21, 1.72, 2.23, 2.23, 2.23, 2.24, 2.23, 2.12, 2.17, 1.74, 2.02, 1.88, + 1.67, 1.73, 1.83, 1.82, 1.73, 1.83, 2.19, 1.84, 1.89, 1.6, 1.71, 1.86, + 1.85, 1.84, 1.87, 1.91, 1.52, 1.95, 1.87, 1.89, 1.91, 1.91, 1.93, 1.9, + 1.91, 1.9, 1.89, 1.89, 1.91, 1.9, 1.91, 1.91, 1.91, 1.93, 1.94, 1.91, + 1.92, 1.77, 1.91, 1.95, 1.19, 1.96, 1.98, 1.94, 1.55, 1.91, 1.92, 1.92, + 1.97, 1.93, 1.99, 1.86, 1.12, 1.93, 1.92, 1.95, 1.85, 1.84, 1.91, 1.12, + 1.82, 1.82, 1.95, 1.24, 1.94, 1.96, 1.21, 1.83, 1.96, 1.36, 1.96, 1.82, + 1.92, 1.68, 1.93, 1.23, 1.96, 1.93, 1.86, 1.41, 1.16, 1.6, 1.25, 1.2, + 1.65, 1.66, 1.87, 1.94, 1.96, 1.91, 1.25, 1.93, 1.91, 1.7, 0.99, 1.81, + 1.92, 1.95, 1.5, 1.47, 1.15, 1.58, 1.18, 1.82, 1.13, 1.83, 1.91, 1.26, + 1.27, 1.91, 1.45, 1.6, 1.29, 1.94, 1.94, 1.23, 1.95, 1.21, 1.94, 1.86, + 1.9, 1.33, 1.75, 2.02, 1.98, 2.03, 1.83, 1.5, 2.04, 2.02, 1.9, 2, 2.02, + 1.95, 1.93, 1.95, 1.95, 1.99, 2, 1.94, 1.96, 1.86, 1.92, 1.88, 1.86, + 1.84, 1.87, 1.77, 1.89, 1.89, 1.88, 1.94, 1.82, 1.79, 1.86, 2.06, 2.33, + 1.88, 1.86, 1.81, 1.8, 1.8, 1.86, 1.9, 2, 2.06, 2.1, 2.2, 2, 2.16, + 1.98, 1.8, 1.8, 1.85, 1.75, 2.04, 2.19, 2.14, 2.19, 1.86, 2.1, 2.11, + 2.18, 2.03, 2.28, 2.19, 2.26, 2.26, 2.21, 2.21, 2.26, 2.33, 2.27, 2.21, + 2.12, 2.23, 2.26, 2.25, 1.88, 2.26, 2.24, 2.36, 2.29, 2.35, 2.3, 2.27, + 2.08, 2.05, 2.27, 2.28, 2.27, 2.28, 1.97, 2.25, 2.25, 2.25, 2.31, 2.28, + 2.27, 2.13, 2.24, 2.28, 2.28, 2.41, 2.34, 9.32, 2.28, 2.38, 2.27, 2.27, + 2.39, 2.11, 2.09, 2.1, 2.06, 2.12, 2.08, 2, 1.93, 2.02, 2.55, 1.54, + 1.64, 1.51, 1.55, 2.82, 2.92, 2.55, 2.37, 1.85, 1.6, 1.72, 1.74, 1.79, + 1.9, 1.94, 2, 2.04, 2.08, 2.12, 2.13, 2.16, 2.18, 2.18, 2.2, 2.2, 2.41, + 2.39, 2.38, 2.4, 2.42, 2.41, 2.43, 2.45, 2.43, 2.45, 2.43, 2.4, 2.44, + 2.4, 2.42, 2.43, 2.45, 2.45, 2.45, 2.46, 2.45, 2.45, 2.43, 2.51, 2.48, + 2.48, 2.53, 2.46, 2.49, 2.5, 2.5, 2.5, 2.52, 2.52, 2.54, 2.5, 2.48, + 2.5, 2.55, 2.5, 2.48, 2.5, 2.5, 2.52, 2.52, 2.48, 2.5, 2.5, 2.52, 2.46, + 2.53, 9 ] + +obs post filters: +# Step 1: Remove Observations from the Edge of the Scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 5-56 + action: + name: reject +# actions: +# - name: set +# flag: ScanEdgeRemoval +# - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + priority_variable: MetaData/fractionOfClearPixelsInFOV +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + action: + name: reject +# actions: +# - name: set +# flag: Thinning +# - name: reject + +# Step 3: Wavenumber Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, + 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, + 7436, 7444, 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, + 7865, 7885, 7888, 7912, 7950, 7972, 7980, 7995, 8007, 8015, + 8055, 8078 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: water_area_fraction@GeoVaLs + minvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: ShortwaveIRCheck +# - name: reject + +# Step 4: Observation Error Inflation based on Wavenumber +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorWavenumIR@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + +# Step 5: Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + minvalue: 50.00001 + maxvalue: 449.99999 +# maxvalue: 100.00000 + action: + name: reject +# actions: +# - name: set +# flag: ObsValueRangeCheck +# - name: reject + +# Step 6: Topography Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + sensor: iasi_metop-a + +# Step 7: Transmittance Top Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + +# Step 8: Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + use_flag_clddet: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + error parameter vector: *iasi_metop-a_oberr + maxvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: CloudDetection +# - name: reject + +# Step 9: NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + test variables: + - name: NearSSTRetCheckIR@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_demisf: [0.01,0.02,0.03,0.02,0.03] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: NSSTCheck +# - name: reject + +# Step 10: Surface Jacobians Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + sensor: iasi_metop-a + +# Step 11: Gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + obserr_bound_max: [ 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 4.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 4.0, 4.0, + 3.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 3.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.5, 2.0, 2.5, 2.5, 3.0, 2.5, + 2.5, 2.5, 2.5, 3.5, 2.5, 2.5, 3.0, 3.5, 3.0, 4.0, + 4.0, 4.0, 4.0, 4.0, 4.0, 4.5, 4.5, 4.5, 4.5, 4.5, + 4.0, 4.5, 4.0, 4.0, 4.5, 2.5, 3.0, 2.5, 3.0, 2.5, + 3.0, 2.0, 2.5, 2.5, 3.0, 3.0, 2.5, 3.0, 3.0, 3.0, + 2.5, 2.5, 4.0, 4.5, 4.5, 5.0, 4.0, 4.0, 5.0, 5.0, + 5.0, 5.0, 5.5, 5.5, 4.0, 5.0, 4.0, 4.5, 5.5, 5.5, + 6.0, 4.5, 4.5, 4.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.5, 4.5, 6.0, + 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, + 6.0, 6.0, 4.0, 6.0, 6.0, 6.0, 6.0, 4.5, 6.0, 6.0, + 4.5, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, 6.0, 6.0, + 5.0, 6.0, 6.0, 5.0, 6.0, 5.0, 6.0, 6.0, 6.0, 5.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0] + error parameter vector: *iasi_metop-a_oberr + action: + name: reject +# actions: +# - name: set +# flag: GrossCheck +# - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + minvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: UseFlagCheck +# - name: reject diff --git a/parm/atm/obs/config/iasi_metop-b.yaml b/parm/atm/obs/config/iasi_metop-b.yaml new file mode 100644 index 000000000..1a4681dc6 --- /dev/null +++ b/parm/atm/obs/config/iasi_metop-b.yaml @@ -0,0 +1,756 @@ +obs space: + name: iasi_metop-b + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)mtiasi_metop-b.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_iasi_metop-b_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + observed variables: [radiance] + simulated variables: [brightnessTemperature] + derived variables: [brightnessTemperature] + channels: &iasi_metop-b_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 + +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: iasi_metop-b + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + linear obs operator: + Absorbers: [H2O,O3] + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)iasi_metop-b.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)iasi_metop-b.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-b_tlapse $(DATA)/obs/$(GPREFIX)iasi_metop-b.tlapse.txt + - name: lapse_rate + tlapse: *iasi_metop-b_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)iasi_metop-b.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)iasi_metop-b.satbias_cov.nc4 + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +#- filter: Create Diagnostic Flags +# filter variables: +# - name: brightnessTemperature +# channels: *iasi_metop-b_channels +# flags: +# - name: ScanEdgeRemoval +# initial value: false +# force reinitialization: false +# - name: Thinning +# initial value: false +# force reinitialization: false +# - name: ShortwaveIRCheck +# initial value: false +# force reinitialization: false +# - name: ObsValueRangeCheck +# initial value: false +# force reinitialization: false +# - name: CloudDetection +# initial value: false +# force reinitialization: false +# - name: NSSTCheck +# initial value: false +# force reinitialization: false +# - name: GrossCheck +# initial value: false +# force reinitialization: false +# - name: InterChannelConsistency +# initial value: false +# force reinitialization: false +# - name: UseFlagCheck +# initial value: false +# force reinitialization: false + +# Step 0-B: Create Derived Variables +# Assign channel wavenumbers in m-1 +- filter: Variable Assignment + assignments: + - name: MetaData/sensorCentralWavenumber + type: float + channels: *iasi_metop-b_channels + function: + name: ObsFunction/LinearCombination + options: + variables: + - name: ObsValue/radiance + channels: *iasi_metop-b_channels + coefs: [25.0] + intercept: 64475.0 + use channel numbers: true + +# Transform radiance to brightness temperature +- filter: Variable Transforms + Transform: SatBrightnessTempFromRad + transform from: + name: ObsValue/radiance + channels: *iasi_metop-b_channels + spectral variable: + name: MetaData/sensorCentralWavenumber + channels: *iasi_metop-b_channels + radiance units: wavenumber + planck1: 1.191042953e-16 + planck2: 1.4387774e-2 + +# Step 0-C: Assign Observation Error +obs prior filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: assign error + error parameter vector: &iasi_metop-b_oberr [ 1.38, 0.81, 0.75, 0.79, 0.72, 0.74, 0.68, 0.72, 0.65, 0.65, + 0.65, 0.69, 0.64, 0.64, 0.65, 0.67, 0.62, 0.61, 0.62, 0.64, 0.59, 0.76, + 1.22, 0.78, 0.64, 0.62, 0.61, 0.69, 0.65, 0.59, 0.61, 0.59, 0.68, 0.62, + 0.68, 4.38, 3.05, 2.31, 1.56, 1.33, 1.58, 0.93, 1.67, 0.72, 0.57, 0.58, + 0.55, 0.68, 0.59, 0.68, 0.59, 0.65, 0.58, 0.62, 0.64, 0.58, 0.64, 0.55, + 0.64, 0.5, 0.82, 0.59, 0.62, 0.51, 0.64, 0.52, 0.51, 0.51, 0.76, 0.52, + 0.57, 0.55, 0.69, 0.58, 0.65, 0.61, 0.59, 0.64, 0.76, 0.72, 1.05, 0.75, + 0.51, 0.65, 1.3, 0.69, 0.93, 1.49, 1.12, 0.68, 0.66, 0.67, 0.59, 0.59, + 0.69, 0.67, 0.64, 0.62, 0.72, 0.69, 0.66, 0.79, 0.78, 0.74, 0.88, 0.77, + 0.88, 0.86, 1, 0.87, 0.85, 0.88, 0.84, 0.84, 0.84, 0.8, 0.8, 0.87, + 0.98, 0.52, 0.65, 0.69, 0.61, 0.6, 0.67, 0.79, 0.62, 0.66, 0.7, 0.65, + 0.62, 0.61, 0.62, 0.53, 0.6, 0.68, 0.95, 0.63, 0.97, 0.65, 0.98, 0.58, + 0.73, 0.65, 0.85, 0.99, 0.76, 0.85, 0.97, 0.77, 0.62, 0.63, 1.21, 1.41, + 1.55, 1.78, 1.35, 1.14, 1.69, 1.79, 1.46, 1.63, 1.94, 2.01, 1.24, 1.76, + 1.26, 1.47, 1.9, 1.66, 2.13, 1.49, 1.52, 1.55, 1.96, 2.31, 2.33, 2.32, + 2.31, 2.33, 2.23, 2.33, 1.84, 2.29, 2.28, 2.28, 2.28, 2.26, 2.26, 2.26, + 2.27, 2.24, 2.23, 2.24, 2.26, 2.28, 2.28, 2.3, 2.15, 2.31, 2.37, 2.27, + 2.29, 2.29, 2.23, 2.28, 2.32, 2.32, 2.31, 2.32, 2.32, 2.31, 2.31, 2.28, + 2.29, 2.28, 2.26, 2.29, 2.27, 2.26, 2.25, 2.27, 2.24, 2.21, 2.24, 2.17, + 2.18, 2.17, 2.21, 1.99, 2.16, 2.2, 2.13, 2.12, 2.13, 2.1, 2.12, 2.11, + 2.09, 2.09, 2.08, 2.09, 2.04, 2.04, 2.1, 2.01, 2.05, 2.03, 2.06, 1.98, + 1.95, 1.94, 1.91, 1.7, 1.76, 1.77, 1.83, 2.04, 1.91, 1.99, 1.99, 2.07, + 2.02, 2.04, 2.1, 2.06, 2.18, 2.21, 2.24, 2.23, 2.23, 1.98, 2.2, 2.18, + 2.18, 2.21, 2.23, 2.24, 2.24, 2.25, 1.8, 2.24, 1.73, 1.73, 2.27, 1.67, + 2.21, 1.72, 2.23, 2.23, 2.23, 2.24, 2.23, 2.12, 2.17, 1.74, 2.02, 1.88, + 1.67, 1.73, 1.83, 1.82, 1.73, 1.83, 2.19, 1.84, 1.89, 1.6, 1.71, 1.86, + 1.85, 1.84, 1.87, 1.91, 1.52, 1.95, 1.87, 1.89, 1.91, 1.91, 1.93, 1.9, + 1.91, 1.9, 1.89, 1.89, 1.91, 1.9, 1.91, 1.91, 1.91, 1.93, 1.94, 1.91, + 1.92, 1.77, 1.91, 1.95, 1.19, 1.96, 1.98, 1.94, 1.55, 1.91, 1.92, 1.92, + 1.97, 1.93, 1.99, 1.86, 1.12, 1.93, 1.92, 1.95, 1.85, 1.84, 1.91, 1.12, + 1.82, 1.82, 1.95, 1.24, 1.94, 1.96, 1.21, 1.83, 1.96, 1.36, 1.96, 1.82, + 1.92, 1.68, 1.93, 1.23, 1.96, 1.93, 1.86, 1.41, 1.16, 1.6, 1.25, 1.2, + 1.65, 1.66, 1.87, 1.94, 1.96, 1.91, 1.25, 1.93, 1.91, 1.7, 0.99, 1.81, + 1.92, 1.95, 1.5, 1.47, 1.15, 1.58, 1.18, 1.82, 1.13, 1.83, 1.91, 1.26, + 1.27, 1.91, 1.45, 1.6, 1.29, 1.94, 1.94, 1.23, 1.95, 1.21, 1.94, 1.86, + 1.9, 1.33, 1.75, 2.02, 1.98, 2.03, 1.83, 1.5, 2.04, 2.02, 1.9, 2, 2.02, + 1.95, 1.93, 1.95, 1.95, 1.99, 2, 1.94, 1.96, 1.86, 1.92, 1.88, 1.86, + 1.84, 1.87, 1.77, 1.89, 1.89, 1.88, 1.94, 1.82, 1.79, 1.86, 2.06, 2.33, + 1.88, 1.86, 1.81, 1.8, 1.8, 1.86, 1.9, 2, 2.06, 2.1, 2.2, 2, 2.16, + 1.98, 1.8, 1.8, 1.85, 1.75, 2.04, 2.19, 2.14, 2.19, 1.86, 2.1, 2.11, + 2.18, 2.03, 2.28, 2.19, 2.26, 2.26, 2.21, 2.21, 2.26, 2.33, 2.27, 2.21, + 2.12, 2.23, 2.26, 2.25, 1.88, 2.26, 2.24, 2.36, 2.29, 2.35, 2.3, 2.27, + 2.08, 2.05, 2.27, 2.28, 2.27, 2.28, 1.97, 2.25, 2.25, 2.25, 2.31, 2.28, + 2.27, 2.13, 2.24, 2.28, 2.28, 2.41, 2.34, 9.32, 2.28, 2.38, 2.27, 2.27, + 2.39, 2.11, 2.09, 2.1, 2.06, 2.12, 2.08, 2, 1.93, 2.02, 2.55, 1.54, + 1.64, 1.51, 1.55, 2.82, 2.92, 2.55, 2.37, 1.85, 1.6, 1.72, 1.74, 1.79, + 1.9, 1.94, 2, 2.04, 2.08, 2.12, 2.13, 2.16, 2.18, 2.18, 2.2, 2.2, 2.41, + 2.39, 2.38, 2.4, 2.42, 2.41, 2.43, 2.45, 2.43, 2.45, 2.43, 2.4, 2.44, + 2.4, 2.42, 2.43, 2.45, 2.45, 2.45, 2.46, 2.45, 2.45, 2.43, 2.51, 2.48, + 2.48, 2.53, 2.46, 2.49, 2.5, 2.5, 2.5, 2.52, 2.52, 2.54, 2.5, 2.48, + 2.5, 2.55, 2.5, 2.48, 2.5, 2.5, 2.52, 2.52, 2.48, 2.5, 2.5, 2.52, 2.46, + 2.53, 9 ] + +obs post filters: +# Step 1: Remove Observations from the Edge of the Scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 5-56 + action: + name: reject +# actions: +# - name: set +# flag: ScanEdgeRemoval +# - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + priority_variable: MetaData/fractionOfClearPixelsInFOV +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + action: + name: reject +# actions: +# - name: set +# flag: Thinning +# - name: reject + +# Step 3: Wavenumber Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, + 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, + 7436, 7444, 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, + 7865, 7885, 7888, 7912, 7950, 7972, 7980, 7995, 8007, 8015, + 8055, 8078 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: water_area_fraction@GeoVaLs + minvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: ShortwaveIRCheck +# - name: reject + +# Step 4: Observation Error Inflation based on Wavenumber +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorWavenumIR@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + +# Step 5: Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + minvalue: 50.00001 + maxvalue: 449.99999 +# maxvalue: 100.00000 + action: + name: reject +# actions: +# - name: set +# flag: ObsValueRangeCheck +# - name: reject + +# Step 6: Topography Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + sensor: iasi_metop-b + +# Step 7: Transmittance Top Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + +# Step 8: Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + use_flag_clddet: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + error parameter vector: *iasi_metop-b_oberr + maxvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: CloudDetection +# - name: reject + +# Step 9: NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + test variables: + - name: NearSSTRetCheckIR@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_demisf: [0.01,0.02,0.03,0.02,0.03] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: NSSTCheck +# - name: reject + +# Step 10: Surface Jacobians Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + sensor: iasi_metop-b + +# Step 11: Gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + obserr_bound_max: [ 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 4.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 4.0, 4.0, + 3.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 3.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.5, 2.0, 2.5, 2.5, 3.0, 2.5, + 2.5, 2.5, 2.5, 3.5, 2.5, 2.5, 3.0, 3.5, 3.0, 4.0, + 4.0, 4.0, 4.0, 4.0, 4.0, 4.5, 4.5, 4.5, 4.5, 4.5, + 4.0, 4.5, 4.0, 4.0, 4.5, 2.5, 3.0, 2.5, 3.0, 2.5, + 3.0, 2.0, 2.5, 2.5, 3.0, 3.0, 2.5, 3.0, 3.0, 3.0, + 2.5, 2.5, 4.0, 4.5, 4.5, 5.0, 4.0, 4.0, 5.0, 5.0, + 5.0, 5.0, 5.5, 5.5, 4.0, 5.0, 4.0, 4.5, 5.5, 5.5, + 6.0, 4.5, 4.5, 4.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.5, 4.5, 6.0, + 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, + 6.0, 6.0, 4.0, 6.0, 6.0, 6.0, 6.0, 4.5, 6.0, 6.0, + 4.5, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, 6.0, 6.0, + 5.0, 6.0, 6.0, 5.0, 6.0, 5.0, 6.0, 6.0, 6.0, 5.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0] + error parameter vector: *iasi_metop-b_oberr + action: + name: reject +# actions: +# - name: set +# flag: GrossCheck +# - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + minvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: UseFlagCheck +# - name: reject diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml b/parm/atm/obs/lists/gdas_prototype_3d.yaml index 1c4d3a95c..b32d4b634 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml @@ -13,6 +13,8 @@ observers: ##- !INC ${OBS_YAML_DIR}/ompstc8_npp.yaml ##- !INC ${OBS_YAML_DIR}/cris-fsr_n20.yaml ##- !INC ${OBS_YAML_DIR}/cris-fsr_npp.yaml +##- !INC ${OBS_YAML_DIR}/iasi_metop-a.yaml +##- !INC ${OBS_YAML_DIR}/iasi_metop-b.yaml ##- !INC ${OBS_YAML_DIR}/sfc.yaml ##- !INC ${OBS_YAML_DIR}/sfcship.yaml ##- !INC ${OBS_YAML_DIR}/gnssro.yaml diff --git a/parm/atm/obs/testing/iasi_metop-a.yaml b/parm/atm/obs/testing/iasi_metop-a.yaml index fed8883c4..477c1c871 100644 --- a/parm/atm/obs/testing/iasi_metop-a.yaml +++ b/parm/atm/obs/testing/iasi_metop-a.yaml @@ -158,7 +158,7 @@ obs prior filters: obs post filters: # Wavenumber Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, @@ -176,7 +176,8 @@ obs post filters: minvalue: 1.0e-12 action: name: reject -- filter: BlackList + +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-a_channels @@ -187,6 +188,7 @@ obs post filters: channels: *iasi_metop-a_channels options: channels: *iasi_metop-a_channels + # Observation Range Sanity Check - filter: Bounds Check filter variables: @@ -197,8 +199,9 @@ obs post filters: # maxvalue: 100.00000 action: name: reject + # Topography Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-a_channels @@ -210,8 +213,9 @@ obs post filters: options: channels: *iasi_metop-a_channels sensor: iasi_metop-a + # Transmittance Top Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-a_channels @@ -222,6 +226,7 @@ obs post filters: channels: *iasi_metop-a_channels options: channels: *iasi_metop-a_channels + # Cloud Detection Check - filter: Bounds Check filter variables: @@ -361,6 +366,7 @@ obs post filters: maxvalue: 1.0e-12 action: name: reject + # NSST Retrieval Check - filter: Bounds Check filter variables: @@ -438,8 +444,9 @@ obs post filters: maxvalue: 1.0e-12 action: name: reject + # Surface Jacobians Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-a_channels @@ -453,6 +460,7 @@ obs post filters: obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] sensor: iasi_metop-a + # Gross check - filter: Background Check filter variables: @@ -537,6 +545,7 @@ obs post filters: error parameter vector: *iasi_metop-a_oberr action: name: reject + # Useflag Check - filter: Bounds Check filter variables: @@ -612,5 +621,4 @@ obs post filters: minvalue: 1.0e-12 action: name: reject -passedBenchmark: 127348 - +passedBenchmark: 144704 #127348 diff --git a/parm/atm/obs/testing/iasi_metop-a_noqc.yaml b/parm/atm/obs/testing/iasi_metop-a_noqc.yaml new file mode 100644 index 000000000..c9ee251a4 --- /dev/null +++ b/parm/atm/obs/testing/iasi_metop-a_noqc.yaml @@ -0,0 +1,158 @@ +obs space: + name: iasi_metop-a + obsdatain: + engine: + type: H5File + obsfile: iasi_metop-a_obs_${CDATE}_subset.nc4 + _source: gdas + obsdataout: + engine: + type: H5File + obsfile: iasi_metop-a_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &iasi_metop-a_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 +geovals: + filename: iasi_metop-a_geoval_${CDATE}_subset.nc4 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: iasi_metop-a + EndianType: little_endian + CoefficientPath: crtm/ +obs bias: + input file: iasi_metop-a_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-a_tlap iasi_metop-a_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *iasi_metop-a_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +# covariance: +# minimal required obs number: 20 +# variance range: [1.0e-6, 10.0] +# step size: 1.0e-4 +# largest analysis variance: 10000.0 +# prior: +# inflation: +# ratio: 1.1 +# ratio for small dataset: 2.0 + +obs prior filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: assign error + error parameter vector: &iasi_metop-a_oberr [ 1.38, 0.81, 0.75, 0.79, 0.72, 0.74, 0.68, 0.72, 0.65, 0.65, + 0.65, 0.69, 0.64, 0.64, 0.65, 0.67, 0.62, 0.61, 0.62, 0.64, 0.59, 0.76, + 1.22, 0.78, 0.64, 0.62, 0.61, 0.69, 0.65, 0.59, 0.61, 0.59, 0.68, 0.62, + 0.68, 4.38, 3.05, 2.31, 1.56, 1.33, 1.58, 0.93, 1.67, 0.72, 0.57, 0.58, + 0.55, 0.68, 0.59, 0.68, 0.59, 0.65, 0.58, 0.62, 0.64, 0.58, 0.64, 0.55, + 0.64, 0.5, 0.82, 0.59, 0.62, 0.51, 0.64, 0.52, 0.51, 0.51, 0.76, 0.52, + 0.57, 0.55, 0.69, 0.58, 0.65, 0.61, 0.59, 0.64, 0.76, 0.72, 1.05, 0.75, + 0.51, 0.65, 1.3, 0.69, 0.93, 1.49, 1.12, 0.68, 0.66, 0.67, 0.59, 0.59, + 0.69, 0.67, 0.64, 0.62, 0.72, 0.69, 0.66, 0.79, 0.78, 0.74, 0.88, 0.77, + 0.88, 0.86, 1, 0.87, 0.85, 0.88, 0.84, 0.84, 0.84, 0.8, 0.8, 0.87, + 0.98, 0.52, 0.65, 0.69, 0.61, 0.6, 0.67, 0.79, 0.62, 0.66, 0.7, 0.65, + 0.62, 0.61, 0.62, 0.53, 0.6, 0.68, 0.95, 0.63, 0.97, 0.65, 0.98, 0.58, + 0.73, 0.65, 0.85, 0.99, 0.76, 0.85, 0.97, 0.77, 0.62, 0.63, 1.21, 1.41, + 1.55, 1.78, 1.35, 1.14, 1.69, 1.79, 1.46, 1.63, 1.94, 2.01, 1.24, 1.76, + 1.26, 1.47, 1.9, 1.66, 2.13, 1.49, 1.52, 1.55, 1.96, 2.31, 2.33, 2.32, + 2.31, 2.33, 2.23, 2.33, 1.84, 2.29, 2.28, 2.28, 2.28, 2.26, 2.26, 2.26, + 2.27, 2.24, 2.23, 2.24, 2.26, 2.28, 2.28, 2.3, 2.15, 2.31, 2.37, 2.27, + 2.29, 2.29, 2.23, 2.28, 2.32, 2.32, 2.31, 2.32, 2.32, 2.31, 2.31, 2.28, + 2.29, 2.28, 2.26, 2.29, 2.27, 2.26, 2.25, 2.27, 2.24, 2.21, 2.24, 2.17, + 2.18, 2.17, 2.21, 1.99, 2.16, 2.2, 2.13, 2.12, 2.13, 2.1, 2.12, 2.11, + 2.09, 2.09, 2.08, 2.09, 2.04, 2.04, 2.1, 2.01, 2.05, 2.03, 2.06, 1.98, + 1.95, 1.94, 1.91, 1.7, 1.76, 1.77, 1.83, 2.04, 1.91, 1.99, 1.99, 2.07, + 2.02, 2.04, 2.1, 2.06, 2.18, 2.21, 2.24, 2.23, 2.23, 1.98, 2.2, 2.18, + 2.18, 2.21, 2.23, 2.24, 2.24, 2.25, 1.8, 2.24, 1.73, 1.73, 2.27, 1.67, + 2.21, 1.72, 2.23, 2.23, 2.23, 2.24, 2.23, 2.12, 2.17, 1.74, 2.02, 1.88, + 1.67, 1.73, 1.83, 1.82, 1.73, 1.83, 2.19, 1.84, 1.89, 1.6, 1.71, 1.86, + 1.85, 1.84, 1.87, 1.91, 1.52, 1.95, 1.87, 1.89, 1.91, 1.91, 1.93, 1.9, + 1.91, 1.9, 1.89, 1.89, 1.91, 1.9, 1.91, 1.91, 1.91, 1.93, 1.94, 1.91, + 1.92, 1.77, 1.91, 1.95, 1.19, 1.96, 1.98, 1.94, 1.55, 1.91, 1.92, 1.92, + 1.97, 1.93, 1.99, 1.86, 1.12, 1.93, 1.92, 1.95, 1.85, 1.84, 1.91, 1.12, + 1.82, 1.82, 1.95, 1.24, 1.94, 1.96, 1.21, 1.83, 1.96, 1.36, 1.96, 1.82, + 1.92, 1.68, 1.93, 1.23, 1.96, 1.93, 1.86, 1.41, 1.16, 1.6, 1.25, 1.2, + 1.65, 1.66, 1.87, 1.94, 1.96, 1.91, 1.25, 1.93, 1.91, 1.7, 0.99, 1.81, + 1.92, 1.95, 1.5, 1.47, 1.15, 1.58, 1.18, 1.82, 1.13, 1.83, 1.91, 1.26, + 1.27, 1.91, 1.45, 1.6, 1.29, 1.94, 1.94, 1.23, 1.95, 1.21, 1.94, 1.86, + 1.9, 1.33, 1.75, 2.02, 1.98, 2.03, 1.83, 1.5, 2.04, 2.02, 1.9, 2, 2.02, + 1.95, 1.93, 1.95, 1.95, 1.99, 2, 1.94, 1.96, 1.86, 1.92, 1.88, 1.86, + 1.84, 1.87, 1.77, 1.89, 1.89, 1.88, 1.94, 1.82, 1.79, 1.86, 2.06, 2.33, + 1.88, 1.86, 1.81, 1.8, 1.8, 1.86, 1.9, 2, 2.06, 2.1, 2.2, 2, 2.16, + 1.98, 1.8, 1.8, 1.85, 1.75, 2.04, 2.19, 2.14, 2.19, 1.86, 2.1, 2.11, + 2.18, 2.03, 2.28, 2.19, 2.26, 2.26, 2.21, 2.21, 2.26, 2.33, 2.27, 2.21, + 2.12, 2.23, 2.26, 2.25, 1.88, 2.26, 2.24, 2.36, 2.29, 2.35, 2.3, 2.27, + 2.08, 2.05, 2.27, 2.28, 2.27, 2.28, 1.97, 2.25, 2.25, 2.25, 2.31, 2.28, + 2.27, 2.13, 2.24, 2.28, 2.28, 2.41, 2.34, 9.32, 2.28, 2.38, 2.27, 2.27, + 2.39, 2.11, 2.09, 2.1, 2.06, 2.12, 2.08, 2, 1.93, 2.02, 2.55, 1.54, + 1.64, 1.51, 1.55, 2.82, 2.92, 2.55, 2.37, 1.85, 1.6, 1.72, 1.74, 1.79, + 1.9, 1.94, 2, 2.04, 2.08, 2.12, 2.13, 2.16, 2.18, 2.18, 2.2, 2.2, 2.41, + 2.39, 2.38, 2.4, 2.42, 2.41, 2.43, 2.45, 2.43, 2.45, 2.43, 2.4, 2.44, + 2.4, 2.42, 2.43, 2.45, 2.45, 2.45, 2.46, 2.45, 2.45, 2.43, 2.51, 2.48, + 2.48, 2.53, 2.46, 2.49, 2.5, 2.5, 2.5, 2.52, 2.52, 2.54, 2.5, 2.48, + 2.5, 2.55, 2.5, 2.48, 2.5, 2.5, 2.52, 2.52, 2.48, 2.5, 2.5, 2.52, 2.46, + 2.53, 9 ] +passedBenchmark: 801416 diff --git a/parm/atm/obs/testing/iasi_metop-b.yaml b/parm/atm/obs/testing/iasi_metop-b.yaml index 2d0654822..660aae464 100644 --- a/parm/atm/obs/testing/iasi_metop-b.yaml +++ b/parm/atm/obs/testing/iasi_metop-b.yaml @@ -158,7 +158,7 @@ obs prior filters: obs post filters: # Wavenumber Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, @@ -176,7 +176,8 @@ obs post filters: minvalue: 1.0e-12 action: name: reject -- filter: BlackList + +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-b_channels @@ -187,6 +188,7 @@ obs post filters: channels: *iasi_metop-b_channels options: channels: *iasi_metop-b_channels + # Observation Range Sanity Check - filter: Bounds Check filter variables: @@ -197,8 +199,9 @@ obs post filters: # maxvalue: 100.00000 action: name: reject + # Topography Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-b_channels @@ -210,8 +213,9 @@ obs post filters: options: channels: *iasi_metop-b_channels sensor: iasi_metop-b + # Transmittance Top Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-b_channels @@ -222,6 +226,7 @@ obs post filters: channels: *iasi_metop-b_channels options: channels: *iasi_metop-b_channels + # Cloud Detection Check - filter: Bounds Check filter variables: @@ -361,6 +366,7 @@ obs post filters: maxvalue: 1.0e-12 action: name: reject + # NSST Retrieval Check - filter: Bounds Check filter variables: @@ -438,8 +444,9 @@ obs post filters: maxvalue: 1.0e-12 action: name: reject + # Surface Jacobians Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-b_channels @@ -453,6 +460,7 @@ obs post filters: obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] sensor: iasi_metop-b + # Gross check - filter: Background Check filter variables: @@ -537,6 +545,7 @@ obs post filters: error parameter vector: *iasi_metop-b_oberr action: name: reject + # Useflag Check - filter: Bounds Check filter variables: @@ -612,5 +621,4 @@ obs post filters: minvalue: 1.0e-12 action: name: reject -passedBenchmark: 124731 - +passedBenchmark: 142616 # 124731 diff --git a/parm/atm/obs/testing/iasi_metop-b_noqc.yaml b/parm/atm/obs/testing/iasi_metop-b_noqc.yaml new file mode 100644 index 000000000..700c46cd2 --- /dev/null +++ b/parm/atm/obs/testing/iasi_metop-b_noqc.yaml @@ -0,0 +1,158 @@ +obs space: + name: iasi_metop-b + obsdatain: + engine: + type: H5File + obsfile: iasi_metop-b_obs_${CDATE}_subset.nc4 + _source: gdas + obsdataout: + engine: + type: H5File + obsfile: iasi_metop-b_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &iasi_metop-b_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 +geovals: + filename: iasi_metop-b_geoval_${CDATE}_subset.nc4 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: iasi_metop-b + EndianType: little_endian + CoefficientPath: crtm/ +obs bias: + input file: iasi_metop-b_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-b_tlap iasi_metop-b_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *iasi_metop-b_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +# covariance: +# minimal required obs number: 20 +# variance range: [1.0e-6, 10.0] +# step size: 1.0e-4 +# largest analysis variance: 10000.0 +# prior: +# inflation: +# ratio: 1.1 +# ratio for small dataset: 2.0 + +obs prior filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: assign error + error parameter vector: &iasi_metop-b_oberr [ 1.38, 0.81, 0.75, 0.79, 0.72, 0.74, 0.68, 0.72, 0.65, 0.65, + 0.65, 0.69, 0.64, 0.64, 0.65, 0.67, 0.62, 0.61, 0.62, 0.64, 0.59, 0.76, + 1.22, 0.78, 0.64, 0.62, 0.61, 0.69, 0.65, 0.59, 0.61, 0.59, 0.68, 0.62, + 0.68, 4.38, 3.05, 2.31, 1.56, 1.33, 1.58, 0.93, 1.67, 0.72, 0.57, 0.58, + 0.55, 0.68, 0.59, 0.68, 0.59, 0.65, 0.58, 0.62, 0.64, 0.58, 0.64, 0.55, + 0.64, 0.5, 0.82, 0.59, 0.62, 0.51, 0.64, 0.52, 0.51, 0.51, 0.76, 0.52, + 0.57, 0.55, 0.69, 0.58, 0.65, 0.61, 0.59, 0.64, 0.76, 0.72, 1.05, 0.75, + 0.51, 0.65, 1.3, 0.69, 0.93, 1.49, 1.12, 0.68, 0.66, 0.67, 0.59, 0.59, + 0.69, 0.67, 0.64, 0.62, 0.72, 0.69, 0.66, 0.79, 0.78, 0.74, 0.88, 0.77, + 0.88, 0.86, 1, 0.87, 0.85, 0.88, 0.84, 0.84, 0.84, 0.8, 0.8, 0.87, + 0.98, 0.52, 0.65, 0.69, 0.61, 0.6, 0.67, 0.79, 0.62, 0.66, 0.7, 0.65, + 0.62, 0.61, 0.62, 0.53, 0.6, 0.68, 0.95, 0.63, 0.97, 0.65, 0.98, 0.58, + 0.73, 0.65, 0.85, 0.99, 0.76, 0.85, 0.97, 0.77, 0.62, 0.63, 1.21, 1.41, + 1.55, 1.78, 1.35, 1.14, 1.69, 1.79, 1.46, 1.63, 1.94, 2.01, 1.24, 1.76, + 1.26, 1.47, 1.9, 1.66, 2.13, 1.49, 1.52, 1.55, 1.96, 2.31, 2.33, 2.32, + 2.31, 2.33, 2.23, 2.33, 1.84, 2.29, 2.28, 2.28, 2.28, 2.26, 2.26, 2.26, + 2.27, 2.24, 2.23, 2.24, 2.26, 2.28, 2.28, 2.3, 2.15, 2.31, 2.37, 2.27, + 2.29, 2.29, 2.23, 2.28, 2.32, 2.32, 2.31, 2.32, 2.32, 2.31, 2.31, 2.28, + 2.29, 2.28, 2.26, 2.29, 2.27, 2.26, 2.25, 2.27, 2.24, 2.21, 2.24, 2.17, + 2.18, 2.17, 2.21, 1.99, 2.16, 2.2, 2.13, 2.12, 2.13, 2.1, 2.12, 2.11, + 2.09, 2.09, 2.08, 2.09, 2.04, 2.04, 2.1, 2.01, 2.05, 2.03, 2.06, 1.98, + 1.95, 1.94, 1.91, 1.7, 1.76, 1.77, 1.83, 2.04, 1.91, 1.99, 1.99, 2.07, + 2.02, 2.04, 2.1, 2.06, 2.18, 2.21, 2.24, 2.23, 2.23, 1.98, 2.2, 2.18, + 2.18, 2.21, 2.23, 2.24, 2.24, 2.25, 1.8, 2.24, 1.73, 1.73, 2.27, 1.67, + 2.21, 1.72, 2.23, 2.23, 2.23, 2.24, 2.23, 2.12, 2.17, 1.74, 2.02, 1.88, + 1.67, 1.73, 1.83, 1.82, 1.73, 1.83, 2.19, 1.84, 1.89, 1.6, 1.71, 1.86, + 1.85, 1.84, 1.87, 1.91, 1.52, 1.95, 1.87, 1.89, 1.91, 1.91, 1.93, 1.9, + 1.91, 1.9, 1.89, 1.89, 1.91, 1.9, 1.91, 1.91, 1.91, 1.93, 1.94, 1.91, + 1.92, 1.77, 1.91, 1.95, 1.19, 1.96, 1.98, 1.94, 1.55, 1.91, 1.92, 1.92, + 1.97, 1.93, 1.99, 1.86, 1.12, 1.93, 1.92, 1.95, 1.85, 1.84, 1.91, 1.12, + 1.82, 1.82, 1.95, 1.24, 1.94, 1.96, 1.21, 1.83, 1.96, 1.36, 1.96, 1.82, + 1.92, 1.68, 1.93, 1.23, 1.96, 1.93, 1.86, 1.41, 1.16, 1.6, 1.25, 1.2, + 1.65, 1.66, 1.87, 1.94, 1.96, 1.91, 1.25, 1.93, 1.91, 1.7, 0.99, 1.81, + 1.92, 1.95, 1.5, 1.47, 1.15, 1.58, 1.18, 1.82, 1.13, 1.83, 1.91, 1.26, + 1.27, 1.91, 1.45, 1.6, 1.29, 1.94, 1.94, 1.23, 1.95, 1.21, 1.94, 1.86, + 1.9, 1.33, 1.75, 2.02, 1.98, 2.03, 1.83, 1.5, 2.04, 2.02, 1.9, 2, 2.02, + 1.95, 1.93, 1.95, 1.95, 1.99, 2, 1.94, 1.96, 1.86, 1.92, 1.88, 1.86, + 1.84, 1.87, 1.77, 1.89, 1.89, 1.88, 1.94, 1.82, 1.79, 1.86, 2.06, 2.33, + 1.88, 1.86, 1.81, 1.8, 1.8, 1.86, 1.9, 2, 2.06, 2.1, 2.2, 2, 2.16, + 1.98, 1.8, 1.8, 1.85, 1.75, 2.04, 2.19, 2.14, 2.19, 1.86, 2.1, 2.11, + 2.18, 2.03, 2.28, 2.19, 2.26, 2.26, 2.21, 2.21, 2.26, 2.33, 2.27, 2.21, + 2.12, 2.23, 2.26, 2.25, 1.88, 2.26, 2.24, 2.36, 2.29, 2.35, 2.3, 2.27, + 2.08, 2.05, 2.27, 2.28, 2.27, 2.28, 1.97, 2.25, 2.25, 2.25, 2.31, 2.28, + 2.27, 2.13, 2.24, 2.28, 2.28, 2.41, 2.34, 9.32, 2.28, 2.38, 2.27, 2.27, + 2.39, 2.11, 2.09, 2.1, 2.06, 2.12, 2.08, 2, 1.93, 2.02, 2.55, 1.54, + 1.64, 1.51, 1.55, 2.82, 2.92, 2.55, 2.37, 1.85, 1.6, 1.72, 1.74, 1.79, + 1.9, 1.94, 2, 2.04, 2.08, 2.12, 2.13, 2.16, 2.18, 2.18, 2.2, 2.2, 2.41, + 2.39, 2.38, 2.4, 2.42, 2.41, 2.43, 2.45, 2.43, 2.45, 2.43, 2.4, 2.44, + 2.4, 2.42, 2.43, 2.45, 2.45, 2.45, 2.46, 2.45, 2.45, 2.43, 2.51, 2.48, + 2.48, 2.53, 2.46, 2.49, 2.5, 2.5, 2.5, 2.52, 2.52, 2.54, 2.5, 2.48, + 2.5, 2.55, 2.5, 2.48, 2.5, 2.5, 2.52, 2.52, 2.48, 2.5, 2.5, 2.52, 2.46, + 2.53, 9 ] +passedBenchmark: 790328 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml b/parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml new file mode 100755 index 000000000..c15df2b62 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml @@ -0,0 +1,238 @@ +observations: + - obs space: + name: bufr + obsdatain: "{{ DMPDIR }}/{{ RUN }}.{{ PDY }}/{{ cyc }}/atmos/{{ RUN }}.t{{ cyc }}z.mtiasi.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" + + latitude: + query: "*/CLATH" + + longitude: + query: "*/CLONH" + + satelliteId: + query: "*/SAID" + + sensorId: + query: "*/SIID[1]" + + scanLineNumber: + query: "*/SLNM" + + gqisFlagQual: + query: "*/QGFQ" + + fieldOfViewNumber: + query: "*/FOVN" + + sensorScanPosition: + sensorScanPosition: + fieldOfViewNumber: "*/FOVN" + sensor: iasi + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + stationElevation: + query: "*/SELV" + type: float + + sensorViewAngle: + sensorScanAngle: + fieldOfViewNumber: "*/FOVN" + scanStart: -48.330 + scanStep: 3.334 + scanStepAdjust: 0.625 + sensor: iasi + + fractionOfClearPixelsInFOV: + query: "*/IASIL1CS{1}/FCPH" +# type: float +# transforms: +# - scale: 0.01 + + sensorChannelNumber: + query: "*/IASICHN/CHNM" + + # ObsValue + spectralRadiance: + spectralRadiance: + sensorChannelNumber: "*/IASICHN/CHNM" + startChannel: "*/IASIL1CB/STCH" + endChannel: "*/IASIL1CB/ENCH" + scaleFactor: "*/IASIL1CB/CHSF" + scaledSpectralRadiance: "*/IASICHN/SCRA" + + splits: + satId: + category: + variable: satelliteId + map: + _3: metop-b + _4: metop-a + _5: metop-c + + ioda: + backend: netcdf + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.mtiasi_$(splitvar).tm00.nc" + + dimensions: + - name: Channel + source: variables/sensorChannelNumber + path: "*/IASICHN" + + - name: Cluster + path: "*/IASIL1CS" + + - name: Band + path: "*/IASIL1CB" + + globals: + - name: "platformCommonName" + type: string + value: "Meteorological Operational Satellite" + + - name: "platformLongDescription" + type: string + value: "EUMETSAT Polar System in sunsynchronous orbit" + + - name: "source" + type: string + value: "MTYP 021-241 IASI 1C RADIANCES (VARIABLE CHNS) (METOP)" + + - name: "sourceFiles" + type: string + value: "gdas.t00z.mtiasi.tm00.bufr_d" + + - name: "datetimeReference" + type: string + value: "2021-08-01T00:00:00Z" + + - name: "sensor" + type: string + value: "IASI" + + - name: "processingLevel" + type: string + value: "Level-1C" + + - name: "converter" + type: string + value: "BUFR" + + variables: + + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [ -90, 90 ] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [ -180, 180 ] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteId + longName: "Satellite Identifier" + + - name: "MetaData/instrumentIdentifier" + source: variables/sensorId + longName: "Satellite Instrument Identifier" + + - name: "MetaData/scanLineNumber" + source: variables/scanLineNumber + longName: "Scan Line Number" + + - name: "MetaData/qualityFlags" + source: variables/gqisFlagQual + longName: "Individual IASI-System Quality Flag" + + - name: "MetaData/fieldOfViewNumber" + source: variables/fieldOfViewNumber + longName: "Field of View Number" + + - name: "MetaData/sensorScanPosition" + source: variables/sensorScanPosition + longName: "Sensor Scan Position" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degree" + range: [ 0, 180 ] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [ 0, 360 ] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [ 0, 90 ] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [ 0, 360 ] + + - name: "MetaData/sensorViewAngle" + source: variables/sensorViewAngle + longName: "Sensor View Angle" + units: "degree" + + - name: "MetaData/stationElevation" + source: variables/stationElevation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + - name: "MetaData/fractionOfClearPixelsInFOV" + source: variables/fractionOfClearPixelsInFOV + longName: "Fraction of Clear Pixels in a Field of View" + units: "1" + range: [0, 100] + +# The unit from BUFR is W m-2 sr-1 m -- this is radiance per wavenumber +# - name: "ObsValue/spectralRadiance" + - name: "ObsValue/radiance" + source: variables/spectralRadiance + longName: "IASI Spectral Radiance" + units: "W m-2 sr-1" +# units: "W m-2 sr-1 m-1" + From 7f0a2952069b6503b962873966c9d3f315a5bb97 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Thu, 7 Dec 2023 14:37:49 -0600 Subject: [PATCH 51/58] temp --- parm/atm/obs/lists/gdas_prototype_3d.yaml | 4 ++-- ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml b/parm/atm/obs/lists/gdas_prototype_3d.yaml index 82cf45285..e9ab7e2f8 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml @@ -7,9 +7,9 @@ observers: ##- !INC ${OBS_YAML_DIR}/amsua_metop-c.yaml ##- !INC ${OBS_YAML_DIR}/sondes_prepbufr.yaml ##- !INC ${OBS_YAML_DIR}/atms_n20.yaml -##- !INC ${OBS_YAML_DIR}/atms_npp.yaml +- !INC ${OBS_YAML_DIR}/atms_npp.yaml ##- !INC ${OBS_YAML_DIR}/aircraft.yaml -- !INC ${OBS_YAML_DIR}/satwind_goes-16.yaml +## - !INC ${OBS_YAML_DIR}/satwind_goes-16.yaml ##- !INC ${OBS_YAML_DIR}/satwind_goes-17.yaml ##- !INC ${OBS_YAML_DIR}/scatwind_metop-a.yaml ##- !INC ${OBS_YAML_DIR}/scatwind_metop-b.yaml diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index a6d4afe98..a23f4bf4a 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -83,7 +83,7 @@ def apply_corr(self, sat_id, ta, ifov): @timing_decorator def merge(amsua_files, splits): - ioda_files = [(f'esamua.{x}.tm00.ncc', f'amsua.{x}_ta.tm00.ncc', f'amsua.{x}.tm00.nc') for x in splits] + ioda_files = [(f'esamua.{x}.tm00.ncc', f'amsua.{x}_ta.tm00.ncc', f'amsua_{x}.tm00.nc') for x in splits] logger.info(f'Ioda files: {ioda_files}') file1 = [f for f in amsua_files[0].values()] file2 = [f for f in amsua_files[1].values()] From f63bfd435ca2d59cc8f664fc52ee0ae594cd0231 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Sat, 9 Dec 2023 09:31:54 -0600 Subject: [PATCH 52/58] a few small changes --- parm/atm/obs/lists/gdas_prototype_3d.yaml | 2 +- parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml | 2 +- ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 6 +++--- ush/ioda/bufr2ioda/combine_base.py | 2 +- ush/ioda/bufr2ioda/run_bufr2ioda.py | 10 +++++++--- ush/ioda/bufr2ioda/utils.py | 1 + 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml b/parm/atm/obs/lists/gdas_prototype_3d.yaml index b06edebeb..de7b722a4 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml @@ -1,5 +1,5 @@ observers: -##- !INC ${OBS_YAML_DIR}/amsua_n19.yaml +- !INC ${OBS_YAML_DIR}/amsua_n19.yaml ##- !INC ${OBS_YAML_DIR}/amsua_n18.yaml ##- !INC ${OBS_YAML_DIR}/amsua_n15.yaml ##- !INC ${OBS_YAML_DIR}/amsua_metop-a.yaml diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml b/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml index 7d6d377a4..d577e94ce 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml +++ b/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml @@ -126,7 +126,7 @@ observations: source: variables/satelliteAntennaCorrectionsVersionNumber longName: "Satellite Antenna Corrections Version Number" - - name: "MetaData/fieldOfViewNumber" + - name: "MetaData/sensorScanPosition" source: variables/fieldOfViewNumber longName: "Field Of View Number" diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index a23f4bf4a..29e9ed83d 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -15,7 +15,7 @@ BACKEND = CPP -AMSUA_TYPE_CHANGE_DATETIME = "2022120000" +AMSUA_TYPE_CHANGE_DATETIME = "2000120000" YAML_NORMAL = True # current as normal @@ -58,7 +58,7 @@ def re_map_variable(self, container): if self.yaml_order: ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) else: - ifov = self.get_container_variable(container, 'MetaData', 'fieldOfViewNumber', sat_id) + ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) tb = self.apply_corr(sat_id, ta, ifov) self.replace_container_variable(container, 'ObsValue', 'brightnessTemperature', tb, sat_id) @@ -83,7 +83,7 @@ def apply_corr(self, sat_id, ta, ifov): @timing_decorator def merge(amsua_files, splits): - ioda_files = [(f'esamua.{x}.tm00.ncc', f'amsua.{x}_ta.tm00.ncc', f'amsua_{x}.tm00.nc') for x in splits] + ioda_files = [(f'amsua.{x}_ta.tm00.ncc', f'esamua.{x}.tm00.ncc', f'amsua_{x}.tm00.nc') for x in splits] logger.info(f'Ioda files: {ioda_files}') file1 = [f for f in amsua_files[0].values()] file2 = [f for f in amsua_files[1].values()] diff --git a/ush/ioda/bufr2ioda/combine_base.py b/ush/ioda/bufr2ioda/combine_base.py index ef08b3503..f2d5987a3 100644 --- a/ush/ioda/bufr2ioda/combine_base.py +++ b/ush/ioda/bufr2ioda/combine_base.py @@ -59,7 +59,7 @@ def get_container(self): file_name = self.split_files[sat_id] container[sat_id]['dset'] = xr.open_dataset(file_name) for group in GROUPS: - container[sat_id][group] = xr.open_dataset(file_name, group=group) + container[sat_id][group] = xr.open_dataset(file_name, group=group, decode_times=False) except FileNotFoundError as e: logger.info(f'File not existed exception for sat id: {sat_id} with error msg: {e}') container.pop(sat_id) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 1ae7db797..976192ce6 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -25,7 +25,10 @@ def mp_bufr_converter(exename, configfile): cmd.add_default_arg('-c') cmd.add_default_arg(configfile) logger.info(f"Executing {cmd}") - cmd() + try: + cmd() + except Exception as e: + logger.info(f' Exception: {e}') @logit(logger) @@ -84,8 +87,8 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): bufr2iodapy = USH_IODA + '/bufr2ioda_' + obtype + ".py" # append the values to the lists - config_files.append(json_output_file) - exename.append(bufr2iodapy) + # config_files.append(json_output_file) + # exename.append(bufr2iodapy) # Check if the converter was successful # if os.path.exists(json_output_file): @@ -96,6 +99,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): BUFR_yaml_files = [os.path.basename(f) for f in BUFR_yaml_files] BUFR_yaml = [f.replace('bufr2ioda_', '').replace('.yaml', '') for f in BUFR_yaml_files] logger.info(f'All obs type processed by yaml: {BUFR_yaml}') + BUFR_yaml = ['ncep_1bamua_ta', 'ncep_esamua'] for obtype in BUFR_yaml: logger.info(f"Convert {obtype}...") diff --git a/ush/ioda/bufr2ioda/utils.py b/ush/ioda/bufr2ioda/utils.py index 76c3b7ba0..3fcac6635 100644 --- a/ush/ioda/bufr2ioda/utils.py +++ b/ush/ioda/bufr2ioda/utils.py @@ -62,6 +62,7 @@ def nc_merge(file_name1, file_name2, target_name): vars1 = grp1[k].variables vars2 = grp2[k].variables for key in vars1: + logger.info(key) var = vars1[key] if vars2.get(key): var2 = vars2[key] From 99c431aa42022e91f12ff082d646fcb34f6a83cd Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Thu, 21 Dec 2023 14:21:08 -0500 Subject: [PATCH 53/58] Update observation YAMLs for conventional surface pressure (#803) This PR includes the following updates for conventional surface pressure 1. Add observation YAML under configuration 2. Update observation YAML under testing 3. Add conv_ps to 3dvar obs list 4. Set window shift to true (time boundary is inclusive) 5. Validate 3dvar for surface (station) pressure (end-to-end) - [QC validation](https://github.com/NOAA-EMC/GDASApp/pull/803#issuecomment-1851387894) - [Increments](https://github.com/NOAA-EMC/GDASApp/pull/803#issuecomment-1852496937) See [PR #792 ](https://github.com/NOAA-EMC/GDASApp/pull/792) for the BUFR converter work to convert the following surface pressure observation into IODA: - SFC - 181, 187 - SFCSHIP - 180, 183 - SONDE - 120 The related UFO evaluation (using GSI obs and geovals) results for surface pressure are documented [here](https://docs.google.com/presentation/d/1OvVMYOutxVyk0GE93Jzoh3hQVoj-dCgrwsVWDDlPT1s/edit?usp=sharing). --- parm/atm/obs/config/conv_ps.yaml | 142 +++++++++++++++------- parm/atm/obs/lists/gdas_prototype_3d.yaml | 1 + parm/atm/obs/testing/conv_ps.yaml | 62 ++++------ parm/atm/variational/3dvar_dripcg.yaml | 1 + 4 files changed, 122 insertions(+), 84 deletions(-) diff --git a/parm/atm/obs/config/conv_ps.yaml b/parm/atm/obs/config/conv_ps.yaml index 8fa4ab185..ffae20073 100644 --- a/parm/atm/obs/config/conv_ps.yaml +++ b/parm/atm/obs/config/conv_ps.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)sondes.{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)conventional_ps.prepbufr.nc obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_sondes_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_conventional_ps_prepbufr_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [stationPressure] @@ -21,6 +21,11 @@ obs operator: geovar_sfc_geomz: surface_altitude geovar_geomz: geopotential_height +linear obs operator: + name: Identity + variables: + - name: stationPressure + obs prior filters: # Initial Error Assignments for SFC Observations - filter: Perform Action @@ -103,36 +108,107 @@ obs prior filters: xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] errors: [ 110, 120, 120, 120, 120, 1.0e+11] -obs post filters: -# Observation range sanity check -- filter: Bounds Check - filter variables: - - name: stationPressure - minvalue: 37499.0 - maxvalue: 106999.0 - action: - name: reject +# Create PreQC group variable (pqm in GSI read_prepbufr) +- filter: Variable Assignment + assignments: + - name: InputObsError/stationPressure + type: float + source variable: ObsErrorData/stationPressure -# Reject all ObsType 183 -- filter: RejectList +# Set observation quality-realted variables +# Create PreQC group variable (pqm in GSI read_prepbufr) +- filter: Variable Assignment + assignments: + - name: PreQC/stationPressure + type: int + source variable: QualityMarker/stationPressure + +# Create PreUseFlag group variable (usage in GSI read_prepbufr) +# Initialize +- filter: Variable Assignment + assignments: + - name: PreUseFlag/stationPressure + type: int + source variable: PreQC/stationPressure + +- filter: Variable Assignment + where: + - variable: + name: PreUseFlag/stationPressure + is_in: 1-15 + assignments: + - name: PreUseFlag/stationPressure + value: 0 +# Re-assignment +- filter: Variable Assignment where: - variable: name: ObsType/stationPressure is_in: 183 + assignments: + - name: PreUseFlag/stationPressure + value: 100 -# Reject surface pressure below 500 hPa -- filter: Bounds Check - filter variables: - - name: stationPressure - minvalue: 50000.00 - action: - name: reject +- filter: Variable Assignment + where: + - variable: + name: ObsValue/stationPressure + is_defined: + - variable: + name: ObsValue/stationPressure + maxvalue: 50000.00 + where operator: and + assignments: + - name: PreUseFlag/stationPressure + value: 100 -- filter: RejectList +- filter: Variable Assignment where: - variable: name: PreQC/stationPressure - is_in: 4-15 + is_in: 9, 12, 15 + assignments: + - name: PreUseFlag/stationPressure + value: 100 + +- filter: Variable Assignment + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + assignments: + - name: PreUseFlag/stationPressure + value: 101 + +## Observation range sanity check +#- filter: Bounds Check +# filter variables: +# - name: stationPressure +# minvalue: 37499.0 +# maxvalue: 106999.0 +# action: +# name: reject + +## Reject all ObsType 183 +#- filter: RejectList +# where: +# - variable: +# name: ObsType/stationPressure +# is_in: 183 + +## Reject surface pressure below 500 hPa +#- filter: Bounds Check +# filter variables: +# - name: stationPressure +# minvalue: 50000.00 +# action: +# name: reject + +#- filter: RejectList +# where: +# - variable: +# name: PreQC/stationPressure +# is_in: 4-15 # Inflate obs error based on obs type - filter: Perform Action @@ -145,6 +221,7 @@ obs post filters: name: inflate error inflation factor: 1.2 +obs post filters: # Calculate obs error inflation factors for duplicated observations at the same location - filter: Variable Assignment assignments: @@ -172,26 +249,6 @@ obs post filters: name: inflate error inflation factor: 0.7 -# Reduce original observation error based on obs type and subtype -# In this case: reduce original obs error for buoy -- filter: Variable Assignment - where: - - variable: - name: ObsType/stationPressure - is_in: 180 - - variable: - name: ObsSubType/stationPressure - is_in: 0 - assignments: - - name: ObsError/stationPressure - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: ObsError/stationPressure - coefs: [0.7] - # Calculate obs error inflation factors for large discrepancies between model and observations - filter: Variable Assignment assignments: @@ -290,6 +347,5 @@ obs post filters: is_not_in: 0, 1 action: name: reject - # End of Filters diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml b/parm/atm/obs/lists/gdas_prototype_3d.yaml index b32d4b634..bcb6ddb55 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml @@ -17,4 +17,5 @@ observers: ##- !INC ${OBS_YAML_DIR}/iasi_metop-b.yaml ##- !INC ${OBS_YAML_DIR}/sfc.yaml ##- !INC ${OBS_YAML_DIR}/sfcship.yaml +##- !INC ${OBS_YAML_DIR}/conv_ps.yaml ##- !INC ${OBS_YAML_DIR}/gnssro.yaml diff --git a/parm/atm/obs/testing/conv_ps.yaml b/parm/atm/obs/testing/conv_ps.yaml index 20a449a8a..6c795e6c5 100644 --- a/parm/atm/obs/testing/conv_ps.yaml +++ b/parm/atm/obs/testing/conv_ps.yaml @@ -104,15 +104,14 @@ obs prior filters: xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] errors: [ 110, 120, 120, 120, 120, 1.0e+11] -obs post filters: -# Observation range sanity check -- filter: Bounds Check - filter variables: - - name: stationPressure - minvalue: 37499.0 - maxvalue: 106999.0 - action: - name: reject +## Observation range sanity check +#- filter: Bounds Check +# filter variables: +# - name: stationPressure +# minvalue: 37499.0 +# maxvalue: 106999.0 +# action: +# name: reject # Reject all ObsType 183 - filter: RejectList @@ -121,19 +120,19 @@ obs post filters: name: ObsType/stationPressure is_in: 183 -# Reject surface pressure below 500 hPa -- filter: Bounds Check - filter variables: - - name: stationPressure - minvalue: 50000.00 - action: - name: reject +## Reject surface pressure below 500 hPa +#- filter: Bounds Check +# filter variables: +# - name: stationPressure +# minvalue: 50000.00 +# action: +# name: reject -- filter: RejectList - where: - - variable: - name: PreQC/stationPressure - is_in: 4-15 +#- filter: RejectList +# where: +# - variable: +# name: PreQC/stationPressure +# is_in: 4-15 # Inflate obs error based on obs type - filter: Perform Action @@ -146,6 +145,7 @@ obs post filters: name: inflate error inflation factor: 1.2 +obs post filters: # Calculate obs error inflation factors for duplicated observations at the same location - filter: Variable Assignment assignments: @@ -173,26 +173,6 @@ obs post filters: name: inflate error inflation factor: 0.7 -# Reduce original observation error based on obs type and subtype -# In this case: reduce original obs error for buoy -- filter: Variable Assignment - where: - - variable: - name: ObsType/stationPressure - is_in: 180 - - variable: - name: ObsSubType/stationPressure - is_in: 0 - assignments: - - name: ObsError/stationPressure - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: ObsError/stationPressure - coefs: [0.7] - # Calculate obs error inflation factors for large discrepancies between model and observations - filter: Variable Assignment assignments: diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml index 64f8ec8c0..a9258d744 100644 --- a/parm/atm/variational/3dvar_dripcg.yaml +++ b/parm/atm/variational/3dvar_dripcg.yaml @@ -3,6 +3,7 @@ cost function: time window: begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' length: $(ATM_WINDOW_LENGTH) + bound to include: begin analysis variables: &3dvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr] geometry: fms initialization: From 30543abd86dfff5a050b6b57e28f19726ca41316 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Mon, 22 Jan 2024 10:04:18 -0600 Subject: [PATCH 54/58] a working version matched with GSI --- CMakeLists.txt | 4 +- parm/atm/obs/config/amsua_n19.yaml | 44 ++++++++++++++++++ parm/atm/variational/3dvar_dripcg.yaml | 41 ++-------------- parm/ioda/bufr2ioda/amsua_metop-a.ACCoeff.nc | Bin 0 -> 12468 bytes parm/ioda/bufr2ioda/amsua_metop-b.ACCoeff.nc | Bin 0 -> 12624 bytes parm/ioda/bufr2ioda/amsua_n15.ACCoeff.nc | Bin 0 -> 12460 bytes parm/ioda/bufr2ioda/amsua_n16.ACCoeff.nc | Bin 0 -> 12460 bytes parm/ioda/bufr2ioda/amsua_n17.ACCoeff.nc | Bin 0 -> 12460 bytes parm/ioda/bufr2ioda/amsua_n18.ACCoeff.nc | Bin 0 -> 12460 bytes parm/ioda/bufr2ioda/amsua_n19.ACCoeff.nc | Bin 0 -> 12460 bytes ush/ioda/bufr2ioda/antcorr_application.py | 11 +++-- .../bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 34 ++++++++------ 12 files changed, 78 insertions(+), 56 deletions(-) create mode 100644 parm/ioda/bufr2ioda/amsua_metop-a.ACCoeff.nc create mode 100644 parm/ioda/bufr2ioda/amsua_metop-b.ACCoeff.nc create mode 100644 parm/ioda/bufr2ioda/amsua_n15.ACCoeff.nc create mode 100644 parm/ioda/bufr2ioda/amsua_n16.ACCoeff.nc create mode 100644 parm/ioda/bufr2ioda/amsua_n17.ACCoeff.nc create mode 100644 parm/ioda/bufr2ioda/amsua_n18.ACCoeff.nc create mode 100644 parm/ioda/bufr2ioda/amsua_n19.ACCoeff.nc diff --git a/CMakeLists.txt b/CMakeLists.txt index 2091893fd..5968b4b89 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,8 +80,8 @@ if(BUILD_GDASBUNDLE) option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" BRANCH develop ) option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop ) - + # ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop ) + ecbuild_bundle( PROJECT ufo GIT "https://github.com/JCSDA-internal/ufo.git" BRANCH feature/crtm ) # FMS and FV3 dynamical core ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) ecbuild_bundle( PROJECT fv3 GIT "https://github.com/jcsda/GFDL_atmos_cubed_sphere.git" BRANCH release-stable ) diff --git a/parm/atm/obs/config/amsua_n19.yaml b/parm/atm/obs/config/amsua_n19.yaml index aa91d3f6c..f6342d467 100644 --- a/parm/atm/obs/config/amsua_n19.yaml +++ b/parm/atm/obs/config/amsua_n19.yaml @@ -18,6 +18,7 @@ obs operator: Absorbers: [H2O,O3] Clouds: [Water, Ice] Cloud_Fraction: 1.0 + Cloud_Seeding: true obs options: Sensor_ID: &Sensor_ID amsua_n19 EndianType: little_endian @@ -100,6 +101,48 @@ obs post filters: clwret_ch314: 2 clwret_types: [ObsValue] +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/surfaceParam + type: int + function: + name: IntObsFunction/Conditional + options: + defaultvalue: 0 + firstmatchingcase: true + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + value: 100 + - where: + - variable: + name: GeoVaLs/land_area_fraction + minvalue: 0.99 + value: 85 + - where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.99 + value: 90 + - where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.99 + value: 85 + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/thinningCriteria + type: int + function: + name: IntObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/surfaceParam + coefs: [1] + # Calculate CLW retrieved from background - filter: Variable Assignment assignments: @@ -246,6 +289,7 @@ obs post filters: horizontal_mesh: 145 use_reduced_horizontal_grid: true distance_norm: geodesic + priority_variable: DerivedMetaData/thinningCriteria # round_horizontal_bin_count_to_nearest: true # partition_longitude_bins_using_mesh: true actions: diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml index abd945842..2fa198924 100644 --- a/parm/atm/variational/3dvar_dripcg.yaml +++ b/parm/atm/variational/3dvar_dripcg.yaml @@ -1,7 +1,8 @@ cost function: cost type: 3D-Var - window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' - window length: $(ATM_WINDOW_LENGTH) + time window: + begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' + length: $(ATM_WINDOW_LENGTH) analysis variables: &3dvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr] geometry: fms initialization: @@ -33,7 +34,7 @@ variational: minimizer: algorithm: DRIPCG iterations: - - ninner: 2 + - ninner: 1 gradient norm reduction: 1e-10 test: on geometry: @@ -50,40 +51,6 @@ variational: field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml diagnostics: departures: bkgmob - - ninner: 4 - gradient norm reduction: 1e-10 - test: on - geometry: - fms initialization: - namelist filename: ./fv3jedi/fmsmpp.nml - field table filename: ./fv3jedi/field_table - akbk: ./fv3jedi/akbk.nc4 - layout: - - $(layout_x) - - $(layout_y) - npx: $(npx_anl) - npy: $(npy_anl) - npz: $(npz_anl) - field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml - diagnostics: - departures: bkgmob1 final: diagnostics: departures: anlmob - increment: - output: - filetype: auxgrid - gridtype: gaussian - filename: ./anl/atminc. - geometry: - fms initialization: - namelist filename: ./fv3jedi/fmsmpp.nml - field table filename: ./fv3jedi/field_table - akbk: ./fv3jedi/akbk.nc4 - layout: - - $(layout_x) - - $(layout_y) - npx: $(npx_anl) - npy: $(npy_anl) - npz: $(npz_anl) - field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml diff --git a/parm/ioda/bufr2ioda/amsua_metop-a.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_metop-a.ACCoeff.nc new file mode 100644 index 0000000000000000000000000000000000000000..28b062added0ed09b7e6e31f046fbe6e7587fa0e GIT binary patch literal 12468 zcmeHtc{G*Z_rHpiqIuBdHc$$gBBGrP(O`BB;kpy9;oj@sOU5##C@RfD8YwiYG^og& zii$*rCY6RKsg#oMer`AK&v&iw=es_?wSND6*UDOteV^w!XP-U1_C9By^O%}h43hf& zCx>J(UPHMw3md9X{BQ>Tm*dh*ed$~-JK*=TQTS{m`5dqD>jWGTn-<7p1qZNcz8s;5 zCkVsmBp>TJRxCq`iK!`%?d?tDvPGt57Bnjx4Q~TYik6mvt}aDeQ%g%-Q%hZ2kD{$@ zsH179sqaSN(}M#vLg-u;jwf2i<9jz$h{so_|9(E&d5sOt0mTXkAZ;RU{;z!G z#qa++FUkAChR8z3ks}KDt%}%wZWI#}TU$zqo`x>PoGuXgs!?nM=pt{PAW)6sz^60W z6gpSL=5py2G%$zBVRP{kQm_GqO=tR2*04o3e039w$r=YIB;LFbHRf+eiUpRn9B`L- zyjB0rUnUNw+SaCjoV5r&^v79?DO!sRHMI;EY5nick~QT82C}&#axl_27fnguu~`&v z0WXk({H?4g-kbo`;$Lm4;mz`C|2g?<`hQ(ohzjOen#PODJ$4Kq1aTU^;kGYAWy`m zkS^wMebgvaUVw&tjsg990ePO_PlOfZ)t z`u$VfXNdJe6Y{wn&4LpUK&1x+{|3z$7i4Nm;;k}#*jW3C1@!S7Yj}a#G z{`4bN`5%3u@o#hy{z*-gKP>*g=``yo%0l9Z)ME%<3-LHWGsG~*~B zu8h6#QuQhjH^LvD6S)IX@_cE(qAd_*V;=TW^p@xIpl*O;;EuwA{}X5tqRHOp2CIv)d3 zJyuY)YB3PyN(ToO`T%iLuqx#1Bb4V$^`}SQQO@ROk3>r#vdi*TOhuX*ByBTg3J^Pc zKhKRn0mPQBW7VqhKqSaU_Z8ETzpIVPshc>SzRfpwBM=W7T`Q}!fp}E@&G)}P@9v4eqlnpE|v?=le2zU<5Bngqo2!b9m9Za};keNpqo1f=l_ zlN7%pwQ?%(i9ouFyK`+S(j=GjUyYEaURzkc7wOo0qcYU6e|_JbultbS?QHEfwBJ)k zmv{ScAllw0$Lh`kq9tvVoM%4}O&^#_N%*`$FEE_iiuM{~cKhjDAnt}{uCsoJ_ElDE z*KP))Vwg{UiYfZz?cwk@xoD>mo2!?izcx#CK253yBKaJDzV0R5U7OG! z$B&v#Li@FH@|WL2d$u{I+(|A%8r1o#;|J0uEVpTS_&#&$ur$O=>mrDJzXSWNzkTUD z;`Pae&}l*F=VsNZu|^4qGgY5Kdl26Z-3y+mw4fZCmKk$PQLf^h=Z|keIr%-8+Yzrf zZHqQt2mqqcf5NwIRY06|8hPmMCm>Gh9lU3ZIL>Sxv+q7je9)j%( zqZLj~&IjTauh``W>RWR*G2G`E;{MyR$up!8NA@dj^HJXWS7$p<$2fhkxM!ce0BMsT zy9oX9aBIP-N{okk!_uC2i;ybC9NVOTG{dRm8Uv}7aP+>1*p6|_^b+Qg2llf@+=}~$ zZyoiZ)zBe09_MJ=ie8L+9n;?V*#1u97LD}Vh}*P`V|(inx3f?7cO$;8*Ux=9(;5Ax zVjoUHzvt;W+B^?Le~t(pV(|osw3CK)$tswCIyN?k6rf(OR`xkMqMe3K)6n^d{MX-d zTrZ1$yuOcm1mmxDTLAH%iE-7MS2c45#@$eZ`WHumczN^k{&6V(t0AW^y%`O}>uCl4 zc|Ab98MMkZ?Klvfwhiazp+7o{p8qI9K5uIR9Is(sd-w3<==NDayia&H=`H5P_gkjd zKfreHzS*`-K)b#(&UH5^!uIx6bv(pJ=QQ7U_Bh_FPFj35>iP1)eqPQ(l;=!h)?9n^ z|B>PGFE9>jE2T#-#5`6-1~TTI()eB04ZnaWTKPh?9s3`Y>6}yJi~bEP5?s58xXwN3 zbif#hg3Lj!Du}NequN(IK)Y5jc2iO?4^@{I)KbgP-<|V*sfF5< z`p18dyyHstB7PbfYUYDED9<~-l<732m<{FkBfX~c&1nqA!Rptel`xK)-RqE zWHFv{JC5k*$YUNzw`;mS7yX*BYl!m^AhyRfm*`Bycz+(`D23x?hWS)?ics%cm*+mj zJbP-Z^v0~~KwKi+_E=+lT<@f$om_(XuGeL^vW<-P&5XE_|ia!^iesN>Ddy~-*`h^d=;(@r98}ps>0`;B1 zF1a6y`d+WjP4WQZkgB;bdmrlOD*JdW_D@b=w`-h0zbmPioaw~4cr@GR1^TZpFYVWK z%(L~%HOrYeUX!|@zYgVV@i49|#5if6w713(ar3nM*!q+6aUL=1Fmy-1y_h6*XQM09 zrr|YO>4>*OW}C~=&K*-;jU1#h+u3lSSzi_S|>D{k_LX`V;_VkW4 z#L;U}Wq;v0#O>$iwrvRy3W+7kID=z~2a9q6u85+K4)a-E#=Dn6h_~Pm`;Q~u?@n`3UONQs-Escc!BcWu||9A4$qmt_J zTQDCt-rN%0>5J>PeI_T%aNcPStw~$gfqA+3)t*SqkFBQfcc*FLI&hXjNA3ZnrWN(? zWs&M@r|WLO_n#*goz2Gkp~2{dQipWo zM8}4yNI%Nm2*LTW<+D+gC$?*)SX8hukG3vJQmn%D*`NECZ`>soxNc~<8(F(ZA8DfH z*y}j|J(2Z!u;wGmD%>n8>@Qo9u_g}umA~rTWgZ7Vm8snk$pF41 z=1};BwcwpT`_gXp*}$q!rM(dy0cNi0C@U)^VCr1<`?&ZyFg|8%4Wz39<9yA^@;j%2 z@#&&JeO)Io+8TD%Y-A^{z#J>lt7> z{%Cq+w*|ZtO`l46g@P}0aY3Yz0Kd(7&sJR-2i&Oro9`d`4*aYI@nbC;A?T%H%oF)@ zAZRJN(6}oGL~~xI?%^v!@RB+o?`7E#d_mZG-sccRYi-$Sv41&4U9fjARapcPLBT0A z_c%b9q560Il;;pa@pW<_{6XYK!x_6DgvlKw0zs&1JbFQe1_+7+yT8+;L9p`UjVbO& zKoDhq?B{xa5JVd2#R^A4P=64sLiH;IiOaPB2wF99eMN#11f^B%9o)eJ0j1w@*zP(I zl!P;d0c{{;ycxn;ybXkVsa1wS6F{`;>}kXC_dt~3H|E`r6bPOmv$Ets7lcffejK+e z7($1BVr<^x2;t;%bkh}xin?LB;dUxSo4(YZ;bjalwvKtKHzq=C(kq*VV~rq2nMy85 zV10{>s&3vHh6bY5eW=*O5iqIU=FY7ZxxIImM!rlea#8*9(Bm$yuo_#opjkv)nqwv!sjHe zHj9J+rHzlwz93F})iZd?oudxXu`OVzyN9@91zy*mfWMzFb+* z?s5S9$B{t%0shldFZ~*J8vMwzOZfwEewq4sv@8J5>Oo%iwhITS@8)mx$H8mF8&MGEvL{Wh@eoAm1?7z|r$h8AqotzL^$=^FZf+jWgYZDk zwvN;w2sv3-Yf|F?qQq@pZs&6$s5A8RQj25=NXhNLldA~c%y^xKidn$ySAH4La0?h| zhaNdyyA5;!A@@?x7Q7BBjo)V+2DJ0Nfkeeo@HC7%X;Wbxl$whWV6L31%!I7mp$Fk&*JSK9S9A`IauCg4iSQw zz5|EOL7bjT%poURh#}$Sd;p@8^7howZo+ztp`w%(R0yl8KeXxSF$kG^gMJc*fM~DQ z@Z=SdAiUf)_seUpnF+8#FS*srtArYxD`ra%SRYP zyrkYtI1+yAZRAymF*&|7N~r@j7_5qzw|gZ-w8lQVsM`#owk|ENvduvFWJ|(XS|af2 zCd_r284%F<$*y>30r*S%nr-d54SrV#RjD^O1IMd(LbJ|3@SXa0@)&Lr_#CAR3q51N zhu59|Ot}|)cAsB6;)WggD&9Ak?ji-8!D%nLb(aCh$L!ptb#=fo*(j6T8V;ODW8Y+YRZ|7TnN}d&n=N9pYN@7 zO38tB^(Wp^L~kMT@`)zbsFe_X^^1aTh$_V9$K-b=hJc_?yIlFrLEwwif)sE&g2!(d zHxvRiSKob^TMRx;uP4XmmjQc5u%7(j6~G+P-gx&rrR1?(}$Y(bw<`*+T<2DR&FI?l-XZ4DHmlWhvlMcX-ATS3U5$v)njUt9)QS z^npvqK2ISapsu3THufIyt!C4r_fH1Fs0HudKa2;_z1rke`B8uM{p;Vqc>Wu||7Y?Q z>(Si(hx#S}ONJk7HRUHT)ORHp&oBl0iQva0fqU5bm1Y%ZzaMxz%KQ7q-?gs@uT)HfHNKY%%KDrM) z43BVrc0U0R$-2pX@2H5(p`*b)IA*JR+9Gh*+v{IU^n&|@;gQktO8@Y0^_V<-gb8@q zq~i+G96Ub!viZ4p570iGt37PS1p1nV9#_+Qfmz~sq}Wefod*V~Rl2XF$n+3dU~2R=JuGG_0n2lmmt?y+aj1G8@MHNmqG zp!d0Kd@yb?crDGAzBZRRu#Wm8uUo+-HM_ls!A&xMx=QLdR|#L&_0$!*LEpfosl=r| zegQZW?RQ7)&;V!hv~WBZoCPDdHW(Fx^Xk=Yc}E!FoL5S0z8nfJl6m{j`pi|bes@z$ zxO_|c4!BF!HD2UtoqiTDZqnm7P-guXNTs>gXc)M7^nI# z;J#}6%#{zA;3la9U3QAD`K|8-XX$=ZlT1x;yjwcG?ywR#*iA^cTDKhRC3TJcT&s^S zn&iMvVh_9R{p~FuwX2xF z1p8gihGC(p;2^0J9VPg48bcNs^+4U*^*ygY2VBYOeCRY-8}@j3SJz7LTzcA9zvnu5 z9nU;trRACD1lv?1)I*A7FQO%PVoOMEFW9;na4ZRI_a9mhcUPH_PV z5qLNjtZXsBKSJ)@Q;v}|L<~P3L}(s^h!HnzpRZs-gxa#&li6($;Xl92+E(kpUaHWoJSkYr=;6 zOodxx7~mZm`G$6K%Rv3ksBXcXng*cPZ}@(JIS0IMt9oWm9S&YfTA#W1Er2FjztNT^ z<|N%!2HNtg-hv-lK+{uEyCT#8&kv2;Z32IS=be>@6b8=(&-zo-F!aDvd@MH#XhZ7A z9SNW*(-tWfE&J!5-1VQV548Ht9$GpZz-!r!=VzAA1h0*s56xBE4PNCvSr^_XgIAAE z&XI5XfbNv~cuUVmU`TMn$a*Kz9wa@m@4)^kCUHkJl&wJ;FZW!j2U{-xb-ba(DsIAD_LHvHc*>KYx)M zTxc?2S9){xo*mg91MAcQJ@{t=Ouu{wrtE1-=UN9~#@HRzk6i;Sd2(Z_8`u(jdv8mL zdUEtE_|A*AtLnk}Q-X6p@wS3B1Wf&XUi>HTcqBQ-E_RfLwN**8)a?WVap5ez2n56<~*lsS5Y9!S)KnyN^8;Y{!xT^AKz#_3{8sF}zu>u?B1& zUzw`t)eAO~^G+K)Z5nqr1#E^58FF*|Q?Qoc(AxQN^NC@H!P+K-)|j6M){^sUYcFHx zFy?-+z7}^PaJ4SjoUS}0n>G(@T|1p;+Q@*N#I6q8`mep183j({=Nbnxz&UH-g<35c zV9lyIf7sO>=+7qT1isw^UgXycroG^i`z$EzMjN;lzV0+An*lBd8W$Tr?i$FG&Zcje zPb~ch@&;9W%svUI5?natw4Uj0TL?~@Y>O`o9l=T9GE#HfTX2%(d#C)-+~*@tf>WFl ziEwZVzlvWPJR7Kkoh18LPWaV8c3$j1e8?gAdu@_D3d)Lg0{#jJXz8J{2SsA2!Kx> zGk@PTu#do(*nzL$_Y;qxfY*G#SAJa~@bD`V?-xseC$Te+JT;8k0sNuAzYpQ7x64lr z3IV?T&Xv{ufxzFQeK(QK1OEQ@S9;P|m&J%bcmaO&MPX_872rq4UwBduzz_Ftczrqs z_}f@pgHk(z-~5rfv=;Zl#&3Jvr}qtlcvaWdjn{>s%FY`6&mPpb;X?PG#hN5I)3%X7g?Qa^byb&uLv32P;J z)%CK%iH;AKz-9GZrT(x=aGABJ;QhI;;QT7eeuigHf zfARb`e*e$pE2g)C-F>%QU>Zux)c)_(yIh!6Sz literal 0 HcmV?d00001 diff --git a/parm/ioda/bufr2ioda/amsua_metop-b.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_metop-b.ACCoeff.nc new file mode 100644 index 0000000000000000000000000000000000000000..fdc91fc80fda2858c0a0a3d080664966a0c01c9f GIT binary patch literal 12624 zcmeHNc{J4FyQf82TamO3Q4}(k3gsc$3sFgyG0Y6Z>@(KvOPf?msI;o24J{HCS|nwy zv`3`0`L=3Ny3c6R?|1I+ch9}&+&}I;C+B=U^L^K6`Ml5jJn#FpSzsqE_3KX#$sl}W zI21eAWy1c#N%&umL$L{_ayWF>ud)$XrXViIXKaLkDWX%@Tv`~5P6=iTMO;A?mW#)6 z(Q>4jldNrQxO4`C!l8?77T8f7T@4s!h9t7FDR~ab$dF9dH#E^VG9r;@n;V&$n;5Mi z@u*=egK#Q`hVzN6I9w{#Acz}I(!vp?q*MY0bgGES$t1 zrlILfvWXsvY+_EHV{T|d!Z{2LmoBmCZzeU&oz9|Dh5em|V3AmU@iS~0T_D7v#jSpq z%iwQW4?0K46;K>$;+kKwDN@{5C9! zjHA*>M!NmvS68y6*_?l9W=s-sNn}%91JVMjh)N2gbLaw;nAYEoO{F1o;Z!Dz8o;8H z0-{Jugj5b`5m%4o;%aR@)5X?ffun~3Nup;G*?>&aA)Amq=sXdf9Y7cK1437<&0!7$ zQ0mu(^FLw9+QY`k#ik!fMsrMOk;VOt^$kt+aWJ$v*~r|;@V^GqFdHtLP3MTjbBS?N zVk5>C%E1tD*(4m_(UHVpq7eUNX9EU}K?z5;ElBQk23>%vr<;?O81##%LpC!r)72ws zN)&G`v635AB%%vABpa?kKo1mSSE#8+a-*eIJJViRpnnD+YUR*Ao^S5vM zulOHst<&E}44i+(x5jTv46K(Z$!fugNa0L+gm^YwfB#M1#XdnA-uCwq|HF5sk;pm; ze976o z${}y9^SfXF8va+dF!(pJ2!Fdx((fVu*JPSvg;I!dB$h+$hsAgsijUZi!|)l7k1Rgo z{Y<=;#8^|rM+u)1K=9toOU+FOLevy!l2idi1TU&{w;m8H-*r73J`9Ki?TTZ%+ki+O zxwLef84#-$xI@qkyQP&6W24YKRFTb}Mh%Ci5aTQa5$h>Iq zRBb%cRNsAqRak#heFx(!wx=^um|=lR&g&b*?i){_iwUTo#alxZ7)U zH@6drdpYUTCvO1azC(%o>>)rr$oQ<-^%;nVnn$!xjsxP+!tf^5(?}nTmVHx+^wmA- z4PpO4pXb{gbinuQo*5xsNT+u_X>mmAvav}12GZ>#_g-uBt%+!J5r@7g=Ez4-M3d`?+TG-CeC<@a^Ja`p`ho7G&px}S_{f! ztG;D0>gDz&WmG8;g{@V&s+vHYnzy#zX(bS6RTTp&aQq9K6=ZfL0C8nVB3FtDM6L9Z zro%}<+(^4tXQ=>0eUG3t#R`Z89X2MJFYB4N{vO{kH;9EK7)fnxT zXsh(27VTEzcymnw+NGg$l{B_LT@j*KhITvA^7;ul197~|=}u-o5Qn5_1$CE!*gcXF zzX|2d9`ua12=%t%XM0{f)~}7*7C0LHPO*w+q6y;a6xA>{z7GflDN735pL4ay4@10P zY>pfjxgBw(HoP;j0qt(NuelG9UQ<}~dII9SbaK|Nd5Cj|W8LK@NWDbQykDYy-fmk_ zh<@t&#c=ZJRHXWwdT6Z=n@Ge}RZxLX>Ri-^%Ud1wL#U6;@~8GV|0&rC!$me|x7!b` zhIt~6Q$AKdLpgR8^y$fGs{~MQ?ZfK{AhgRh>(B9QYHN3Zb zBR?x$mV8G2-e^6VN8N_F!)fxdUE}n!3&&iM9tZ(B79FM5ne`r*ifFCrGt4k{{WL6lS>um*SJ)OB&PTp4UF*J%_Bp>`Tlx?z zFLmD2HD3h8$w#9uaFCCp_$fEiJAl|5Z>>{-`r2=CO5Wr-;v!-3wa+twINFu9tos|{ zCv^4fZ1fl9Zy$!Iqns7*{g&0S5O?z)6_a)&g`Q1gOOW!6at7f(@%z1^aDJpyChF1Q zKpM~xFM{^4BXMXCR>yK()K8f??p&crt;C)d{bb17IqiFp&nq^e8+QOvqBZ70Jo>j2 zJ=AyWvAj_J-GSQaXuqy=D!I5G4y~)#D&Gr4UJhx(ZR9hTJ!#~6)Z2z4a@rPqAd((D ze}73D_g7zZ{1hNK-)B$Ei%0!Lq_?P`Kd4>%oxLA%{o5}#G2=*WtcH-haLf@o=}vot-ub=YKff zz5wlVJ5=iDeJk{LM5wQw9`2(nj@VldLma%w_{m56)yeKs68fV&&dKXXsiVD@p$GOw zeUDEHu;rp&ch3QN)aze)f9-pJ?fVzcf7kE-eSdY+c73xP`tw__#&|Evy(wgjQ)Gu;2$?(eC6hWq^! z+pVUnt|JAfoWq#cJXs#5pq~AwyltPf{j^8eek)6ny9DX+tb{i7uWc2+k%{P6pJ;Rk zEyjJ~$%fvd&Syx4rG5!GZ`(KJ2gW%_)vvA7)JJ~q-V9xc`^%%~ZQn*=d~okk)$^>_ zKY7pZeZJ00>*Ty5+<%&!UCRK%kll&7q9slVX!iD#Op_j=@4`NL5}s|D-e74>-4XO>LG4a&hb*E zMu>Y|QS9vrkPtk2&d9D+kVx9~;LF)^NZdM@+>!qPlI{(yG^Q?tlwoVB^T=h8l6DAO zYJ(x=`nc?a(H@W*_r~_cIA2ITBa`tmvJg@m&&(UQ(FRiAUHF*$i3@=>yRG)Ej)#DA zGpn<84gz)2OyBa{Z{Y7YL0L8?5h#{xac9>CzdSF~?-s7$mv!m8&51|g^YZ=Tsx7C$ zS0l0~t^Esl@6I`+zHSeAJD+a*{QW!lXzVI6Q^)`xLTf}p)lBfaHm*ba)3NmpkeSeX6{5_xSyXpYm^BDWA6j4nh+9a zxO%mNAB5b}o4f0EC4{OvsB2sfhtRP6g|e6Gfi*ex&Vo-jfwf%Qd$9LyU=^7WBUC;D zOU66!=2HTqSV?VaMMofF_0ahgk_LnwywWUF!w2Dt)YV%a_YQV$jXwjW15W<|@)JOXS zv}%7`csdYp)7pEiW*t!P6`$F;X*WI;y)uZLiwn)iqCCvx#mh@|U}dtyr@5ZH zA=W_2a6xDY#7Q-sQC;2+(WlPnEzaHtQR1vdejh|6lpRQV&;?;~Ylyz-^MH@R_MG4! zz+QZ+FRFVmgd{jJR#GMaGj5r^3I7W)%{11YPfmg0TM^}7;XMRP;_%?}jA=2pHy}9L z$>A$U1%eMQ+5FS_00j3jr^(A)2IfL(e|NW;z_f4J7ovFzn8!D}YR2OD(-##7mX$#$ zKWu|WmIrVo>ww4UP&jPF0lt}y>e0(@L9pmo{3seTddAA*br4~s{Jm=RBv@$|adPWr zO^7S{)@sl|f`rx5Blph!2#I!2D_fUWL3k@?w!mbzy?vFMEVb@qU;$bcbrgOpl z`&8guCtp;1MFZyd>1&wPY+#1HzfMz?24>KL6K5itAQ+vF#qhDfpoT7KFQNhcy{(>C z?~NlLupJ1oVBGV<&Cz0s66#V$Oj+)al}mcY#1BZBUk08#>TFNZzq+ zVmlLp7fze}GpYud=zPNpiy$OrrB>kjYzQqm?3|Nw71-@el?5ZTfag2nz(&3%2q~9N z^{5CSToP|bMamyuu73emw(YBX#tYaYjFYo}+qpg|0Qw7v@l8dNP5YmK48tPs4y*w0l;O zSz~SjZNZ9t9am;UKzGWqC@1uflPXUKEsci2+{~Wb0s;c7S5c4iss_pf7F&4t?0Eh~I;!v5?(rQE85AO*?F|9Z z66G0k+-l%`8h@?u$$enozWFe1RWYzPpXnQZycgK6x#$sXft51zfUFrESVQKgwAPa$ z^h9KOk*71TtjCBY0=CNc9as4_19o8tS8wj=(gn7k(;TDlFks&-`;pE42<#7richbM z0&d%oGoKD1|Juu%cE0Wefz#WH@jiJV9NVuz5XtVdB0EGt*w@@SMNTCU`7<1AGCv|e_=UbG_TffZF6@QD=Gx0yZe@+-c$kftv4yu1V+@CHoM) zc)s3s^QpiHr^$tGse>TpYV-CblOX86*a2UGpf3v?AH3FvV9aiAw+SGmd5rJjmO;Sz zJXc|I%?IG?9eea~ZY&6Fww<`FC=Wtz+`I`@(tqXsYu~?kMt}Qv{r>;(S4_Vrvz|}5 z55fK0>3m?EFFx3k(+Bj~y9?sR%z!}5h7N5p0{*}eEEfWIVO_aEH7PxkPBpj2+mKjfwYl=g{PFUsEy%uk&g z%ozV%0Ro1elg{JJ1iH$ovbp(gz&!QvRbSOyV28BwU-mr*-k^4)jk|4u_vX@hugx}) zQh3|e+PxT3RAxuLXnYCDAyy;G)=YzBg*Pq>GxZ^9@0X6F3W1Ph@@l&(`vD|&XzY95 z^9d4D9SY9JI1on#oV?*wT~`PJ#;wzL2E+ih zKRwnNzzJo0!?(DS3h={2WXr*P@NwRx;dC<)RyT#U?<9T~7fc}eg6JMQ4qe>`mo zZ2LKRr(;}zg~AQiKLMe5$!hGY4I!mnFU_t@0%jx0GO+70Fu8{eRyWr`Fj21L7nBJ> z#~;Ft@rWB0eF|1-R-aQff2p6#-qvs7^L8(9z^IL{+b&nC>)dq`i<0h zLu$)bU9LaZsaZKs|<$eN?=&ShgDfD?K{Owo8H)SC3|MYaCGseT? z+?GFjo)2N0@1=cy8w9-KzE}Ekm}g|fhSCcT0dGZ5^^O%Z;AzKXIIF}1_u~(C+?rhA zjZA)U(O3m|CesT(pDqL*-oo9x69GJlAL4ym{(1EH)4At{U)1bqBj;6hpz@Q-fq zo0wk={I655zU({?0#0_iYpVqaBqih; zx$HkwwG@N}(+!szQ3vujk$wET#m_xJ6j>8|_Q!b;)tx!+nN)%6I|26DX zh!7~OZ<$*PktbfJ3Y8gu^!Hnjk?x;A=3E9|(fgGr`mVr$K6x!O z-c8;N{AYIWmZ^;bf!D4()xFIi*m2yhp=LS=EltHYwII9__1=UY0HWH>loz5r2*ZyX zX6@Js?9zxMMh@=4j=wui*zN}GL4%)Jrr!saWPOK9&ZR;m-vxxMyHW*>jSylnwqOYF z5QHeaOOT842Ij-lkrVj1AKY5}eW8jMgzQ*f+P3Nfg!ZQ?BOvs8p0q}=A+Y?{wA?JY z0j&HMp=J3GV2N*|+%yOBPxjn%4)5f)!$AJjuom#<4cOg%uHUn>fHU~>(A4Okz^QZE zt29UhxKZ=v=Q%wE?t|uUiV>y2Q>l;K72yiJ#j75;eN+Noqxc}CzQVz`^F6Tqr-b;N#?+x@L_3P1_6B4(>kGEV+Y3Jr`A7-9@Go`M;&3Q z7t}$3rxk93TCfst38i)zLTpgkl$eLL5adF?u%~SDfPbaS z#w4UndJKV*bu*B^om(pYG6(#3Pac~yFADr5`t^G(K1dw_eyKl(?EC%}{I-88o6+V4 ze!cA4VLe=+NY;lx?k)i@27`Z&_~6G1{!<*HWIK=az5=LoW*ss*n*r2*cQO1Z_&kh=UY!*-@ z`;Y&%k@1IzID(J)mqbudFs$1b#=(72n9R17FP6-ad=~AHHk7h;-cp689~g22VfxsDO@{{vjz2pa$Z literal 0 HcmV?d00001 diff --git a/parm/ioda/bufr2ioda/amsua_n15.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_n15.ACCoeff.nc new file mode 100644 index 0000000000000000000000000000000000000000..bf3a9c0f6e182db3d17a9425c293105a961d63f9 GIT binary patch literal 12460 zcmeHNc{G*Z*H@8Jq&ZD)1Er8zL^~Ox!90iSPPm5ay7!W?Oeu;=vyes#jVcW)GAB`y z$k3$H@FbN|^6uw$`M&R3@9$l|Ki+?SYh|s+zR&ZVv(Fwrd!MtKzzDnU5UB#HKKKESi8F$e}W6I4-uL3fKWG;X887 zStQFPYU)cgmMT-!k)bBCrKqTBtEoBI8w;C>N!qgfSv0<|lQ=#i^(QY$M;4EdtjSh` z&q+Gkj>X{z@~9R}^35OJM$-cL!89sIUHi{7S*LYYR67*O-=DOHu<0O=T5KXMeE&cB zh~5v^$M+KUY=QqDHH5Zvr5G7mTT?=GRJACkG@igmnPTlv6LYH( z4u?iTkw(#pt4MO~X>w9d{!#R#8=C^P;zQ7Eva`H-uGi@T+B#i?D0$3aYIT-1e3&y11SWJo+FED_D z{4FdfUTlBVV$hDNUQ914sqZq1Ez66=LxouS6nj<5xg2lhlvS)izR*mh<(N1|K%js{AzjPn zcq>yJ1N~Jsb(JYxuBy74ri!|{hR{x9@IUD%84(6VfgpT0h#N8%$Tp+>13BJQ4ow6{ zq28i?l>gpu43ouY@Yq~ZT#^xKpV@2CACy2Z3SWp3ipW#q{xA7Q87&m{ zQNniro7{5-Ju!sBFcAiEAW8_EwLv(MxZ~FBJBB(692CKg*uQzivC18Oxlk3Qbts+D2|11|6M)C7%d#^ zN2>C_`a<CD21YV>KB=H)B*J!-Rxq^%(5^K_U z$>22(h`lCSFLcs@NZ(o7R$Kza@!FVSJ@P>0U!HM}dKQR_7b$jbnLu22yGS*R0^-_) zORto!192<-(FK7U5G5~`56D>qQ8s=`PGL6?l``9tiada*^39(mJ`0Gu>LqgTm_Xd~ z?uYfBK-^!;2{`l$i2A9Q7W%mW@nE5&rw88Gnf@L>VFR{nklIT82BK!gYF^h9AgU+u zs@5z8qFny)utIMjZu8cJe0z-Ye69Zc_y@|_(&C{!K8uE9sQap1T$J4d?&T0bUVUtT`l?D)x%YQ!LHv;iwr_vcK zO(34~+zda40@3y>TrYMw5YLjTKKEY*;`!GD8QoKXcu{yHJ;N1Ki(Y;f18VT8nd-}Ir?j>Sl6?pY9Nv?a2IK90U~u<&&^Y_f!J?8V@_!Z z+N<=nf?+lg$5xOXfH<-D`pqpnfjC+AU}R+w+I88=8WlevawfY|9dq#BZ{wpG*ze@q z&`14W@SZk2%t;*IE6#W9^}u`Q%>D$l`^kjYohlW`FXzzSnTV6)-Zvfe%W#~zXPtt? zfH)djSEF4EMAjZ^##alZ(~G9(px%T2J7h2CR_P4HK27t*Z*Bmw%cXR`Kmqk>oo$zc z^2gjYyLMjyM8vZ>AH8jW2w9STqG%6}^Latt!lQ_b2*$d6^y{U-!mpe7h*z$&)7dV> z~^E@QgR%)|GYGe)H$UfQ%F^209dxAE?kABfkd zmqKR-p`TlnX2cpKAkLJ$hwVdrH}))ks@RHhsF`OhC`Gx7cV9fY4dvwaUhP1<-nK5< za>*ZvLchu1cT@p!-eK&K+Rs3o);WCN5OJK@HvYhwLbP|CC2ji)AkI!dd@}^w6-LXP znU)X4oxoz}pQvxm`NVMV6NvloE2hmBM;zI%y30j*A6%d3Gz;VO;nLm%wmhWGyzC( zVT+L?a6I<$_Emit_nO9ii?IE@#BHkScM-Q~87KBPAa3WK9_T@Q-E3I!YK{~7OVKu* zf_~4_vA22=fc_j4I>PKJ5NW6N>ys5R|8#C{2`NCmUa#)Aw?{jTnyIS!3HfilW4}=n z{dn_$<1viCwjKV&2L{GfTVBdWd$dpzo!mU>>S2EvR!WLw|QI{H-_!`HNNA9ZE#MJkWZVHuWF> zJ@$$#*^l^XqAQyYXQMptxnjmMkzzKKK8W;&=68qj7zb2uj`*E6dqnR@%-0{hH%MYU z<#ry^&5_1DkZ#j_a{>A_Vb2JsV?gYTYbnv3g7N+$$X*P`%MA0b?h>HhcdjmYgn9PN zcJa+wH-Wf9xbCyW__*0cNjtp^^If0y?tQCq{*f;&X6;0J?8V^?N;qG1JIakm99PQQ z#d=Iax-KNwc@$EuwYr_??}{?TaULtN-Tg|X`{>8As$*-j?jYT(olu>Gv`A*c8-1in zSCvOjLpt1N=UDXHovnxn2Ox^`+T?yMM*ZR@aP}voA9M>JcE&qead1ni%j!0J#vg?^V;DLL1Laq)Pb_e=C&eO}t{S(s-V z6lzv7aJ*&}-atLd*XnLqS%`7cF?D~9KH}zC&xwtv7vVf&(5df+etS7p?A~S3Bb{YtN+GinOrE4$!`*B>ILm8WzP#$ZXNWCz>9X(+B zG!FAwea8D&L5R2Dj|Wd8-fL$%E36-Z_U^p+`|z2+`u_FrUp)Vf-~XI^^|+*Z(l*S; zO}DoNclqG@?SRqgGMsl>LTl1CbYfmEe!VXe^JAOwhrMa)xDK4F*O_|=sc}Wa2T7#5 z8tGb_@cox5Md!2ee&n}3W;n046&E+&-;C7u&>6Ej+mC}tH*%%g3~lH{szeKvUC`kk}P4A%{5-wgC4filn%Uz}3qgIFy3Fnw z5MrB$#|Q!(R5crL`4HrcS&(IJ*^75jC97sPAlCFv?kLW0erJp*Eju%(Q>*q}5I zwnj1*e29MtL7wBZtl1}kOJDb(`_wN87${qnu`Uk$6uxQJGERc8;*6e%WB{Kr^C{fI zI`B%LcV(~2JYZI*Qr`-W0VCIVoP~uvFf_0Fep>nh=$|sS2hfy(ez9hC`MtA1|9shx zwxJ8??TveCaslXq=Zlo*34l({s!EnXe@5_c7xx2Wio4#vzBpixd*3R({T#6De>Ok1 z*#=&T#?Qn&L&1l!v>=jCfbZ74=WDJ_1Wwe!tq+d;0B+Xe_zC7s5cEnv=Be}r;I$TA zYT6S6g88pf_i^PQcv-!-*NSWizQpgk=zRpD)wk_7JGc^}F4?-3Dr!SSP;kneeRdG0 zuku4TrfuKM=T5amF41esU*?K;SDG$}X-@1zvGL&ktHO@K%4iHQntP@S;pl z{MzUTyhuHrSpHZD8VF)mD1Cz07tiLpZq{-Es}0qHdXQx|<5o#;-JHdm2KFwSAt_ttk+j^xA641Otdsa3mKbu(4G_ zNh|LhL_9Cpn@M+s4ek>kuXC{iUXF3}R_pX3zx)24m3{7FDzI1GKk7TT0KAKfPK;0Q z0597j1D)|&z>G7wvoVnOk6)GO_nSUd08NBn+E`@|<)_1drv846W3V&O+JD)HJ!%5F zX8WQHdL+=%DFHjIfib#bpp7>knAXJ8^mV7eYupj{i+UTuXX;f~C8L$V4xgX6)+7@A zUuUOX!j1>v`}Nx54(CJQ zH<1M5Pw<&6L*ADZvwO%qreYSq5os>2OpW^+mnEX}1 zuDDUMJNY&EK2kQwe19AK%AA>|^S*(9Qn=Bps7we@I9(u+(rNHSm7%N}d z16%~x0@uPJ90#uxZbU(t^S(5xrXvug6O=cpoCeWr43-PdHbAUpx~XY=AcP07cXXx( zLCER)I-?pp5G3yKbiJ4hL0zF|mzyPne@gDay<9o)V#I4UR?G#)fWj;P#ydbyJM!4! z#$BNC2&q>(*5G+qe$oNMFrZ%S3m__vgNJ_9X{%~uSbzAyl^ID{;EKBwb3Pvg=kV3$ zKR8pskt3FJXlWxjn0p@Pb#wZ4F3`Q*7Ld%l9R^9`56;rigJab87h@_BGe zFpulEo(rB)yAY9?Ko`Z$pj;faVOHCmQ-H72cGbfb{Vd$>(S%U{oWteKrVzo4=|6Pj z0>tS!#~g96h8PlFPKO{mDQ{m5^)_rY8!1RxlVpDW|X@pNVIV|LiH zrJsRcfN`Ck&tvfI-Wu?Iu{`)`9bX$Uyb%HlH{@!V+XC0AK2&AtD&Q^C(doG`2?SeX zKXv4+g^=C1mz6$Hfl%1hY1Zus;SRzH0XAOWpx3ja0b)wB=27;ALfk6(u$5yBAzoB( zCL9aD^Dgo_#2B629VOohoAlO1EZn;qBHCgfU)E}YP;2Mb*V!h(f4VKUwTT7d1@H@QXg0Qk&!H*Gwp2)vKe_=O%Z;2qeL z|6HLDy!T#QKjxMV_{crbo8>G9?BQuId$d*n+uP*Al@0a4Hrgza+!hY(NJF0$>-E8R z)UA@amhIr@yk`2j%^dJQxX?9qW-SER-D~BD--duUQ3UTG^6IH(m#Ebcef_J9R)`YB=EvlBC58a6U!z>%?P1^w(}EaqI)f)|nm7{t z)z;R&$}I-(<~P$~^UHuWJ6K10_$pwG>1cbfwH9b^sCBEC+y~F|zu&1y%?E0ym3{qN zfAGjo>UrsN7u-eh%q)`*iD1CA!+<|XpP!BI5N zIn7qOeBtaQaMo4z4cSKpS1~s;o1V9@{v7qpjTI^2UVn71ytmRYRlR&@KJX&gTQ?)t~r zzj~g6yJ+3ywtrki=15s^3y#_DmZlAEI{W>Ki9T?fJUTKuUj85cZSK=Xk1+ywt8`pJ znu7bs-&Vi&?*r<`3w1|L7(iRM#Ql1DA23SnkChvn0W;0f=_mUDc(XRx%_=Yhw(XjJ z)jdAIDWIsNcN>Dg%+2=W&NKeuMWs1Ze%v zn;%YG3ZBce#cwQN46UOE<#jE%qHMGOD7cE|PZv@B<|5+jve9vsR?v5FZZ2_dh+hm& zL`UtIU8>+jo)%8xfD>=*_C|w3a9X>zJ?|JDobpPEtye?ASu}4CuFqUV>vvbVgsXSN z?}3|WUE@if*6C&e{WdLrlj9s<_SJen-gE|h&{&a2CjhmwXI^NoGYd$(0kk2!ifNEr^Nm+nLiT-``em7?oc#+1-5&f z^ut0^!A?{s+Kce#FrF+h8o+UX_m8}R9B?74^N}-Qeb|%H-QBCfWBFMh-QJtvnOk6K zUG@&>>DMk>JWK?pXq_C9&iBdpi5sc|Bg}WECN_6K#GKOLPMt*%9yNcOw7~#`OQi7& zl5TSMgBF5aZezA%H5y~s-PG`46gx{iWQ`_kfAzFV%TnJ4* z=d=bQ@o;t;q(WpndrRDuEQm7wv;V(w;`!t}g=E;6n!0@bW;$%*l${HGq6V8DFl6qG zr-N5$EcmtAF7G$UT8FA zS6WN;zFpbwL+jKbJ-Fxmjlcc?hU8gF*Lpi(#Mm6yja>&!X>wz#2UsF}d+kVxdV2gk z_$-XJsp`f0Q-pJ0;kJS$_|N!rUObq0+>`8Mm)eWN`l_V4DmJ{KxNs8XH^=F-$jSxm zO^1 z*#}ml^G+)~ZJKyK1*}Gm7;$^!Gq4ol(9-Ei%c)UE!O|*(+LWIMmZI}(OHV_mFvdZ! zyb*UQV67Hdovl14nYIwDUAmm+SV@45$gXxf25!8Z69o?B=NgAHz$t6Vr8;#9V9u?% zc+|xWXwN5W2E5w`p5)gH#(m(P`#dP@Ry(*BzUk5{n+?u~nwIK6=^o0HPR4H;PtE;? z@`j`En0+cZig4kO({`?}eF->hu`a&Kw+9EF^H{YV@4!Kn?;Y}GIWNYZ28TF#65-$w zejUFwcs^7IJBaqL9Pq1w?7Y~2_>fuBu?QT;Uo*(IHUP))y_I&w=HR4jzi8gAAK)r< zi#d3tct-P`I)N^`)Xls71u#GAsm|^-0G?Fe5{c~55VTEN=IOFV;NB8`Kmc6wnEA(^ zp?w6d$PQc?-(PqH1%WLO`lL4$LLh!c;`MSF1d8k&NS+!-?E>z|Ki`LNRXU_+1cdyNtjl7AAG`oJ`ZB+?=NfP$<1ale2jGVLHNH6;1Kb_V z?Lnzsz-{^DxV#Sc!6xl^(y#L!f&!~K znNyt~uYmK~1@Z%7mEb&AyWqoxZ{YMg%66HKE;xzmUZ*V*E2)$@LpXA3yzH75Z2``U z4yxUh{s7L#caATaT@TK9dMmLx0-PsjMQAUI2d7^llBIH4;M6YH=-aXaoJ95LVE>`} z?wqR~w*<_6ZpZV{DBNVqN8ax^dXRC1Aw)#w|WL1{gJUzmp2E zZhJF{JUIYHX+Z2XS6l~*>aU@D(S9!@tGpoou^%uFkOxQtU~EY@npmCy4089^ZznK^ uEid)_5Ce?-^&btXt-w_L9h8*t7#Lh~W7!j!>47>#v?DN!`W4mQtN#~$(hNNS literal 0 HcmV?d00001 diff --git a/parm/ioda/bufr2ioda/amsua_n16.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_n16.ACCoeff.nc new file mode 100644 index 0000000000000000000000000000000000000000..efeef51c7d41e7536d4d6335b9a395d7d1dc322b GIT binary patch literal 12460 zcmeHNc{G*Z*H@8Jq&ZD)1Er8DBHGCi4QAI6t~=oxuIt`Q#xkWSD$PO~DKx4ysK}g( zibRGcm4+v&l#+Kpx6Ai^*Lr{N`u*|#^II!xJ@$Q`=bU}^@Y(yEea>TIYCc5b&z}sE zp?D4F(#)-?0@1^n_+N%gGx4EwxorPG&&J@h(d2WyCamXkglt+sAS>9PP4nRhgn|4p zd`|MQnrq22pctE&1hT!nXk50?#MGQ-X|3+1uR+n&)Ys9WXlZC_s%dDdY3Wk5v<$R0 z3^epyDLi_xzj_Fr%fj)5%ejGcy1I8@2xTsEl#rOj9-^iLJnmx?S<(Cd z$w&Ntr~$s0a^ML4|EM9doh!xI*v5tuqN}b$F{AT^KB^QOf4a~skRPB*vFFj5Yzmz# zWOKQ63L2NgxJz<>ZYS4a*<`sKU{={Giu;>8aP zpdf!sONtlAAGH{;qq-N%i$?0Zlw!yBV)IcUwgJUKow7Dil`>ybUteEEl`>nbji3#`;B6;1x!ANM~X`_BJDGC4f=x;=tU8TFhUXgioy*JU?3{VW)f1tT#oS1 zPf?#?R*Q_u=Q1>Nj=w*Z?jQUIKm)iV-;uu!F{GCxl{TiaNhsk@QGZf?^6&qWe}wUT zQ6Cj-_rJ+KYrqqOC=3&05C@`!aMJ^`Y7 z9KUMy5+KSI4-F~w2I3Zfb;!5JD9_jG&yRngoXySdi55U)m*uUThBPxs(t7GtAa?bB znHPT&h^<}6t5xEGNRW>1E2blV7i;Cyw{Sc?>+kGFARaclR90yL@woiw13?22Pj;xB zw$=vXDc{ZLV<-@1r15f- z6}}_2bS&_WK)RZ{Yh5bRBA3r2GE}jDec#=0`;p(BY^^r5-!n#+ zSNljH+TJC{>dXeBC2fq1M?VlvADN0t_`E?kAe`EY_8Mz?=h-_T?uBNqw|bBERZ?x& zY6hZWgm-?53HsyRk?^;x zhM>JlUn?19195aY*#U^-d#>HsyaR|6We2ndq8CEre%D!L^`!-Y7Xi>;JBoz9<2aw^)y_YHxQJk`%}2jp3@rS*QGj^mIXj){ zL|neK&ayQ{{5>0SySp8T`dMl55p&W1Q{E-opk9sXFS82}k4^HE(8P!XDXT9oP3Vsk z$4n=q{aQKsD{iAb+ZipgD6X{Zx>-0Q)pE+$r8sepOF+_gYiTyU*x%>n1 z`t)Mx^dR(ev&yts!vw^ciuaJci0_8(g-?}RP!0`?jCrLfSMjb3C$^%Tyq+uVh}T;- zMVl}B15xNV@%#2FAkI0CK78*p5T|qx-8VuUXSR;rf4UIuU28?(_5z4AQxDw;!FGkw za;K-{193aB*!d^wTXQZk-1|7<{`>MNGbIs6b}R4jP~HdE<~Ys3IDNRJXTKdEX%j!Y z2>tPBTfyl{jE8!I(w_H=kt)U<-zZWfdiov2ZCb|hef5ajIj8!&5nngz=e?Tcg#J>t z3#Xvp^K>1oUj(2(M}-bEe+op}DTBIXWz0Vvo0>xkP_Ngk`Wzh4P9vtPYkxxi8*V#n zkVZe=*iSu*@z=WDpZLJUxN6O-nza(+Zn%E^%VR*ix^?Bic$EM3urrt6$^-FcdVybF z4-jvMtaeE|0Ys-w!}kIteOKZ}shyBdFo>zLQxKRPAfJ{yP+3GXMr!@T%m>x}w` z*zWyzo3@E)*Y`%bZu&*o-ma=H5b@DD-RHd>j`zBg7GI5ezIu2dFlQ0Ub2c$+o*nxC z=*ak&7zeeLlJbi%k5!R@jCrRtez#S_Zy<_Ry;Nz({)ePG=hpb3e*=p6*DoQia}PNl zGy^S_L1O}z%{5kC3LxNV5vDKm%l499%^!F#V5mlyhoU4 zPj8dllyw7$%Y^G*D~yjDos_gwOEKT|+U(l93g;ii(qi@wq(@&ITCalhMHf|JEaJFQ z(LUB=0@Af1xy~by>a5Z0Kz~=1DUb13f$i>Bs@z9EmQ@{HlXV;Ep2Z2(Nl1(2#=S8> znsh~V_!Oi=eRhmSzun$~h;Rg=IImUV*Fw}UZX9=CGWtQU@L^Xx5Z7~KesEr*z7yFc z4?Qsej6p?odwMwNvaC+(B>)fgado^>DJaB2b0BZeIYZs@m{lO^tMazWZO zvPLr<@pjmBOF7!PW9sYC<8_eMxz-K;i1hia%522d8;ctk&(|Zp_l;kOa=*!*(UFEY zdLyjtFFcR9{nFgF9pmQpIQQB;=-&=Q;ZKJwpJ z&7Q_#KC8=k|0)Ra7X0zR3B>!o>CQ^)hM~PXF8n@p`mer!{reZsf8+N*Ctp1-sh+SE z^Ks*?t-+l>xPIGje5wrRo#xP*wDldBmy2KTjl}%eYVu)EnkKFTXX|(59z<$VQU5_2 zsh(E4&PIIyWm3_(Y`h=-ZMQkjYpun_4fi)8wL5s)ycYTNZl1an^MA{pmNg-DNH(V5JDqNop+_!w^E;Yw>L(9F$+QoWE6D`Kw z!1?c~wD-fcpHN;YVSzN};l@dMoo94$9`Dg>qCdlR+i!*7&v$U%Auc`5+6Zw&I9UM=86wDKxl=cUznzNUg%H;X7TY{)Ab9mwV%xjLAgm0F{_5TVfdp|_-fPwbrD?RXD@dbRaf zUDF`Q(8mjBD&T#rRehLo7=k{Y3Q!h~00Fry4JrcR=!%p>zPllqtPU3#LCD){pPq)K zKv>So^X5ycV1u@D#*gq^h%sxlWv^vGtiU?<>o_lnU!0eurz{N#whMOmODMzUGR{K7 z(mdD_$(;8g{viZ;j?uB<90wj_?Srn9zaXH$Y-PsUIPg>YrhSij0(_OHbw?xv_>7uM z;T6_`SNfdGd(`Ftt2&kTR(KScxh7*QEfs;OeZ}|Fk{7`El(8*!_3LU60N515nO_4oG10c*_r7TIlQf$i|K z>9Orr@Jcj!CgB+hKFlQrkpcpIx8yxveRVu=qYi9&aQFxCvKGdVvuK2%R|YXpWzPe@ zrRZYg?idixeU-YGrvSlA>%6^|XG8EsLFWbU!w{{xb(i^p6%cjN&aG5=F+>Cfr_9=G z4`BvsKlD;wKnTUh(Vp-Fp(_n%?0yg=caR7Kfr^p*!U}cZ7YB6zphp9L)u)?N-Hrl3 z%Ix^B4Sv9n)Ypv_jE128AXbISHwY4yYyJ?ldeVl91Vac)tJpWRg9Us_zr%<r$gs}L1+(_-VDRERctr8U#j2x4p;@>Fh4g4m?j){Dj&LW~lXT#&$q z7AX~-yt5GTykJizg9_{2$3I@{Vh#Kpljtor>4Sdv{XHZ5?1fa|th|53cXk1I7Z)8L zo8Asyc14D|V|9QPXL@@>Apak~sxa<1eyRYv7{Bzu?9kmyyiuk)qvo&^t0ePBM20+rGj5ahke5K2jKhl>cV#C zgWxxw1maKdn~{3?_lPs#OO{exop;UFZ!UDvp7NLDVVhY7_ngqPDnh)&AeTSL1A8_b6bQ&v(Ttg8O-=3~bpWZq5sDHsOAeVQYa zS4M*1O4q!vZxHV;ndD9v1j&rwxTMSk0^NTXHqKcw=nwyQ1B8E0VK;imaEeku~In{-#Em;ZtrMkM^=O=)0 zOYEoioHY=#>(awe|Yl%j+SgBx??3Zz#mAR18}&$_V1c z^=87+@Z0YquR)CQiCs~O9k5Y|IW|0#k&f?PtwP9ThATvy*8vut+^REp1l*BwfBS1w0Bd+a*M$G7+p~45d+?V z-TBXzdck|og>|ED+JcY51N|A!62KXn_Oe@NIdHs9&tG0&2OQ%~Qpv61z=<^SS-#Ey zd`H|YnQheue$K0>p54R+{{!<~Q>Wj90Q;P+{jE4@7gJW*PZ0B%R{gpK2e zgTKa_d#`ef!Mo|rl-T?-V9yNJl^wbgn4{WTA8fe?^f$EHRg3O}=eghS)MVxYt;5=( z?yWy~WG8jM^tl7>;`nh}FjQ-#_ZDy!mEDfuI=U-bCUPA(DJN^i$&3Q3>6kf*d`X~+ z=Q*dDDwocmnE=ju>b@a+Y2Yg1W^UX47S^4moxZ+21>Eb7%slF%3!ZmZ7^P~K56*|) zaQXO`sRa1fRkYf~-UpuL99r~&DZn4I@coBJ@gTfko4h(d>aV_k{reZsf8+N*CSS20 z&pU9qZz8ayc(ImKe*r^ncXIJe6QG|AK67T%ZlLi*8_(d5tH_T5%TKMhI6_{LBY8Eq4Bsl8xYYh~LAdOM|=Q^fIku`@!Ad zDCbxAQ*alro80z|iO3u-4{pIR+uYI?gPZO?zha^n+$N5UjE-0QhkvX4l#!#1!QDC? zSCD4l{_(f<4f5dixm#=D@LA z-KW0W2e<_kwe&6{@Rz&Mb__az6R_QS*L4~2-WiiIXJm~Od#1=xw}8oPOxpI$b} zfUVdbwmZm$>shef{Q4GG;R@L1X9w)NAV2utZl-7=0J}-CKTPLNhk^cf7LVJN&0c}s zZYP7V&{VJ&*NF~d{5gygZ8?(6!I*PjC}WOY7#I;;zOGP0{{6?iN=H_2G9f~BdF`p}Hi+o0J7nW`FkoH;g}*oTrow8&Xr3t=q(ajoh-cp-(hm;{&GL z?Xe8-3XObAJGFJNerHs-;7&~g(Cat;xX7Ffo_AC{GN+9MPesiy+y~}B6R+QB%Mx>v z?kNFn#WgSf&n%$nDyv=5FF4H;2}De8w0dqb>xl& z(3EJ46^fSsb5HL2PtgNf{T6pk?Tz5M{N{_Z%VvS+rZ0!*sqO*K@}8`VACkedM?2@} z_x(V3OntJo=Myl*IALVH7itZW9Nc$cemR+x!SmOM?gZDL8&1yhF#=fDhYt z7cVIvys-#hyO|GOQc6QVR#XA2ME#|LLLM-y3l;Zf8Ub_Wt&84{;lRM^@AS`wz|bsT zeRB1U!Td&#@?)ROkpg{ULp5a>_NUC!8X2YyIHv;HPue*jbZ45f3OJuqWzkLks(1(qziG1U!hF}}UFr$jwH zb`E^zN848QVErk^xvywj!3zAR{W&il$UE*y4zWudBw<}u(rh(b{$N}I|}S z0S8mpBQMJI!0v{{aYO~!;$f=X{cNzg%JAxAPXn8AWWYQEYjM3iNK*`Mm8-7>>nB&I zDR}mRwfMZ#8c&Tjo;> zzrnmg6&bpJ?$_|Y6LncF?t@L({-jU$I|Kz*U0**z2ZAa)Z~kh+eL9J#N5+dC{@F)6f3tZ_ zi46$W?Zlm)9uPdMDj2_?37#GP=Z3Aw1y6DPHxQpw|b%f=8YG);^6X#Wz zD{?10K3)drHS-kv!z#gf_TqvM=f8o|>nOXWx_aOwu6vy}ORb<$W)0%Vso|1qTC^oN zFF2raL-qqWo7_IOXl5NajK|DD93Xv{V$O5M}g$Cc|<=`Z)M+f>3 z-hVf>Y<{l<&fpni0|qQlue#Rgp7F?UqSN5i_LtRbn{df?XWcz5T=Q-AgS>)*e4 z{u{slG5Lz=C1?ATIYCc5b&z}sE zp?D4F(#)-?0@1^n_+N%gGx4EwxorPG&&J@h(d2WyCamXkglt+sAS>9PP4nRhgn|4p zd`|MQnrq22pctE&1hT!nXk50?#MGQ-X|3+1uR+n&)Ys9WXlZC_s%dDdY3Wk5v<$R0 z3^epyDLi_xzj_Fr%fj)5%ejGcy1I8@2xTsEl#rOj9-^iLJZiCttmys! zn9=z{A61HtKV9e*$PZAZ*z@R2Higa= zvbkJ31&zyLa@btF1QcvQVbhsD6f0|EV>L}Zit$=|M>S)79-_+p<3y3b(iVfR5{{c+CB=*5k6H}aQQeE>MI-fHO0i>mvH7SF+koPrPFWkMN|~>zudlD7N|`NI$5^bp z4P7W?^SKn0Kt7+%Byk~_txB;pGc%(&@abFu{m(N>AeXX|9Vif)iL@LG#|Q`%vMHo% zIb3g53N_GQU0Y9;!sDrHYG|uzYHEq>Gz$Ndev%VoKpY67cLTT~V}Wcl(m#;vP2%LMd1bqFc6hwGYP3+E=Tz1 zr>M^`t3}4-a~YaB$KRhy_YeL9paI;G@5tYV7}Cp;N*mMIB$V){s6Qz``S*XxKf-vv zsE-P^``_fAHQ4&0Z}QpEvd)@h$`Rw8Im)AxT9I3@Qww< zUGF|v=Ly99HQa!MuYjnVd~v>?3lI;p=f^)#&gN$KL<=CY%koxELz)>RX+3o+5W9N6 z%!@w>#MZ9k)hh8oBuGd171NQwi?#CUTR5Jc^>=n75Dyz&Dyy`BcwGMTfuI41Cp%P5 zTWbUHl<#KrF%*c_U*Y<(LxFgfRQ0*<3J}k~?$79&48)7V!|55WK)jT{q;Ya0(s;Sa z3g3}hIu>|GAYIMfwJsHDlJkXchDcMdFDl=Mblm+h8LHU7zVGh0{mAc5wpJV3?-`@Z zt9>L8ZSRs}b!G$6k~T)hqaTQ-k4(iReBPiN5Ke7HdyO@{^XwfE_d+w*TfIm7Dyg<> zH3LyG!aG031pV>uNch`aw9}|9)yvRdTO>N4B~=5Fe4e*JXEP9~Tf1+ZoC(A}i)ph; zL(pEOua%6lfjGLH>;S~^J=bn*-T}mkvIoN}gV3%^SJbHa0g*G&okq>Ud%q2jreVJm zZ$lsTeZhPB&@d-Se6Kv0+T(%u&Y67)X!jEduRGK#kYCQhJ<|~<$GmSi8kFHUv(GpM zNdR#qw6dE z9F#xij``L5LLefZ&HCtV3q;7G^y5Xlah%WdYUdw8TtqO}=A&OP1{QwZC_udOoSn{e zA}(KAXW5z}{+^Av-Q5mE{j9Y3h`H$hDen?(P_M@Hm)Qk~$0qqnXkx^Hl+_oPCiKUN zW2TeQeyyDR6}QozZ4N1SlZ%iBb^h-7iF7H;b$TAY&zv?Q4e`>t7$QIH#C{v@T>gQ0 zeR?r;dJy`#S!G(RVFKbz#e2wJ#CJpY!l%kDD2Ikc#=KILt9aLi6I)SEUeA?w#Op1a zqRkimfhhEw_APe#zVb9Y0vw`NEKs_ZD?$Io7ta7vo;rq;~iHZ9}$zIw#%oKyYXh_4&<^IpwzLVqdS zg;UV)dAbhPF9Oh?qe6$7KLsM~ltEpxGUlI-P0b+%sMqUNeGU$2rxDZDwLc;M4YwUO zNTVNb?57^Z_-oznPkdlvT(#y^&02|ZH(bB|$-MVsMJj(xi*qO_3<$-uJy}&Q8 z2Z*;rR=cE~0HV{T;rx8`M`zKCpGC;$U5&rPbObw>R| zZ1?`VP1{7Y>wBYIH~k`PZ&y_pi1_H7?(^Ob$9vsLi?2pKUp+h!n6n7wIh&X@&kp^6 zbY%QXjDy-rN%=*X$EwIc#=KJ+zuT(eHxNauUaGWX|3gxpb8CFizX3)3>z5GMxrZDN z8Uay|Iiyt?@pW@d`^txC*9yiSN($zo>e7N*Y8m>wbN+ATQOIAS%Kl&?`sIPnyR^yw z`0ue-T**GfPa{LsY$ylidC!wDnT`~*q3i*q*R{Vpj>R}w^G03~_9Tq=7eNjZI9_I$cXg)_^}c;&-XqMj zr?*LN%DMr>Wx{o@6~@PnPDM;i0ipAKR7Q@--+y! z2cf9%jq2PacOVX{mPK=AkbG%=o|LXG6e$T)> zTd!2Jf{Eibsqy>kP`(y-qsl^zllIB`Y77uJ&$^FqIJE%h5yK7xH}u=f$r5)rxgc#C zS)-Ybcsp#mr5x?tG4=K6@j6KBTRL|(!USK)jgQ8u@U95!HLuh^V^aA zW>4cVpVei&e-(sy3;uZE1mgYPbZ4b?!_eLx7k(c){a4?={{4&Rzw!H@ldm3^R8QE7 z`MB}c*5FPbT)*u%K2?VEPIG8Y+WHR6%f+wvMq++!HTke7O%vCFv-LZ245SN~2ZG^ZXo!9P6 zuY#CHMR~7{J0UtKcFsDhW{7grbXi8b3=!nA+^HMF-_FH_whDyxe6Nn#{~1E(teA@{ zd$ED?AF!T$3@DmWsgCzT*37$qQh7%Gee_R|Up}npNd@&j91| zB|rN5PGGb(?5@cLUIQ#>6SqtOGSu{e>D}$J)vgd){ zQgpF#cMJ&UzDnK8Q-I*5b>3dfvmyARp!0(FVTjh;y372)3W&OB=T@q`7$Sm#Q)cb8 zhcE-RA9^V-AcW%MXixZo(3OTWc0UM`J4ghAK*dOYVTC&IivzlU(4&FB>eJ1sZbyM1 zWp@1620!3O>g&b|Mnh145UWDv8w82UHGc?NJ!wNlf*}N@RqPwu!2&*|-(kd_I^dUt zGX?%_AYi;5##*u+1pBB}20;@+xcb}~g9-ORnBY71{mv8!o+!1dua@ zhks^l+3En{qX-XAx4QxE=XWQ ziYM;yCOr~u{ywtGrhebkpGWgRT%dhKUDx-j9>a_RS(stLx8UNeiSv>8R%`l9Ks$o z0z`dM(D5d@0ZQo*mI!#-jA1MvNNbz!^n zLGT+-0`VvK%}Blcd&C*=CCe_QkHGnD;_cqD5IAdwc-q-48l*m3zSB?0nt@O8zE>>4 zO5jvnFWHs+8hjtA8fLz~1%74DEVDV^z&|P6_*GOU1Sp*<5XuW6pltNZ&e~Go_85*< ztn3CJf@_{@;UJEK*NQfxAk2AhnoQ$ih|&$pn@~=N=+%bHglFm@)+*i1EItsz1323| zQiCAmR9&rcjXekxw|lx?$c3QJ&@;=-lfgeFxBqUg0(ddwwHqpC1G8W0m4Cx+V5A*> z?0EeS(D{VSD_tA#Jft{bzfl;_F7yTv701BCAnKHLwF#^{wEyz7q%3g7-HBPB4}f#{ zDvKZ7NkHXFq#Rt*0FD-3hi6tPfFtM24o{sgKrNp*`$YFwaCW}#QaIEAT(!=r=}bHa zZV47~eKxbfGioOyG7}i$xEYX(t1-lSt8)qnbX%`@xT2p$+dbM4>YsC{yvYnA_%VG4 z51)rPUFVp?jy4cO!prF(L?`9#t)bn54d%mzDJ!WER#ks^^ReR)GVdn+6bu95KFyKI zDTTvBENf$qNx8|SPT^oM`bLRU&lIrzCUe;i9!1wU51 z14s55_yw5M>iawf->xkI-xn%^pU$y05kngwpm2SzmW3Vgoa#c=maGK+QeEBd^AkY0 zCH7N$&Kd~Wb!%zq12qVRogL;~R0wwzO$e~z+Is!&<@FF#k~N32Hx%MlDu%5XWd!l! zdNbi@`0aO**C59D#I7jC4%n!_I%58wRS?k{`}mShGlbeWx4h0a1;Nv;3Fl~uz@r;8 z*Joycf9Gf0;#~#cC+TClt>+H-UK>)S*4zvn&)$j6+WWz0+Pf)Zxkcc8j4mkjhym}w z?)>LUz2Lp)!n#p6ZNW$3f&L6<3E&J(d)ckC95~*l=P$3X1CH?~spQsh;6xhvEMI2; zz9Vjy%(iL+Kj+m`&u-#^|AG0gsnhR4fc@PTuH-EUctdWSxEP^G;o!0Gq>%23+v9(PG4W10`7H3W*&9X1<$)Hj8Zkr2j@d? zxP1J}R090#Dq3w~?*q?r4lVk?6yT3p`2NGAco5#NOH_&*Zjc0JjRpic87P$SqV#%;P3~s69 z!h8?7MZFrK`c)O&!Y-^btvCyABZMm|e&&Ol7CZk*$wqKH#P8wKrNLctdYRU-{ornJ zl=G|mDY%Q*O>X=bKuym z?o;3G1Ka|NT6&ie_{-gBI|d!V3D|DE>$(hh?~KWqv$G!9$MU+zox1?cx}n$k&qILT z=e+6R_$A=EEL-yWJm%m!YCvAsg3GG5`;LICc>Z(|*KaOjzAhW6D|LdtgL6}fbA9|m za3b38joPUWPULCf1THx7M{jE|ECi=DYufUTGQcUXl-P156r9EL_Q3kgMZA7@RY^(@$KetnCpa0P7hvjg^BkRN<+H&ZkbfZe3nAEtAs!$5yKi^uKCX0O0* zx069wXe!u?>qG}J{v5}W1x7tk_jUcq>(2ofvN|6=9oB_C8QImf3Otsb@zLwK0iL-9 zRyJktfRTRnlI6ohV2Rhs5$OV-e4n_%IxxawM`~hIJ4DPX4erog0O3({r^p)iL%39$ zpg{HZV7(nC5FHpmSk|ZG#9J*0JLS34z^aHpmL==B?aTx8A#&pRp}nbSssr=sQ;?gMk6iPvwmWr;aS z_mqIP;+hx#XBN zXiBuj3PsERxhHr1r|1E#ev7-N_D1kre)GlIWwXF@)0e~ZRQG^qc~92G56R%!qn&f~ z`+lH1rasx)^9dMYoG`N93$=zw4(>ZJzno0UVaoyQOI-EOrNM*y6db=0-l5}bz=v(S ziKU|+*h=%Ud9 z6g+#uT72GVji*iH&!vF%h+)HSZFmM&VjNmIJ!w8U;s{t-htL}H^T0}cer@Gxn7MD2igd%=_ZdcmX@+;g7?h23le*TOfQ`eif0`C#J`gC|{sdD6+`E%T{` z-(cRLijLVQ167O*$DG!)y={xYakEYF6@ddd@|{O(Y<~xi;(YIzFVB53`V=_EDUt{W z$M9?TrNQ&TI@nRXf8~f@4P@uV{=Z{M}F$BI$?;@$}kr1?1R_^K22H@QkeLw&_@|gL@ z?!kQop4bjNIp1G+1Oiz)WZPmJ$$PNVFf%ezBl316;h(34$Ui2kFY4=s&MaExzS`NSq_iK1_CI)!h zS=)kAJAv2yiMp&7_rWG?f6}M>9fAU@uCJe<13{IYH-9zZKAl9=Bjd#m|LmikzuCN| z#0CWGcH&M?4+x%B6^!4{1kVotbHi5Tf~UBC@_5qLRGzd8+{Ja~I>PckwX+h|iSw$< z6}gigA1{ORnt6)-VU^%KdvU>s^WVVfb(GywT|ICT*S$`grB=`=vj%bG)Nsi)E!q;C z7aY*IA^QQGO>Q4sG_wwz@$^<|Qv^6q%!*jNARe54g-DkwWPwwgLW6Jfa&QvYqXYd1 z@4uT`HosQ_XYsr`P(M0f(c$c#VgoMkm^-TEqhVcA){xX}J#g!Gyu0(`slWRE_3vLi z|Bc`On0&?blCyo_nhQ(=N$HxX>A)~hsR%o95g6ig7{=xjiCdp2z<6UIpD|qbpK~Ab ztH#lnfsxqMc5!$MFz~CugJosDvh!;Ie>MQ`>BQ@{TG25=Nq^1z$jqW)c#H? zz`E_t1oGqnn56-+S6y)(D6YQ-@5TGQ%&hW)_{V<0+)o}L34yse-FSR?0x-$lU%wr| v8nUd^^Fs_U^VfYeqO|}^`FBuK!ee0a$c<%BV5JA@646v(74<1=yx06M^)C!X literal 0 HcmV?d00001 diff --git a/parm/ioda/bufr2ioda/amsua_n18.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_n18.ACCoeff.nc new file mode 100644 index 0000000000000000000000000000000000000000..72310903c39fdd141d2fe864264b6fcd795a839a GIT binary patch literal 12460 zcmeHNc{G*Z*H@8Jq&ZD)1Er8DBHGCi4QAI6t~=oxuIt`Q#xkWSD$PO~DKx4ysK}g( zibRGcm4+v&l#+Kpx6Ai^*Lr{N`u*|#^II!xJ@$Q`=bU}^@Y(yEea>TIYCc5b&z}sE zp?D4F(#)-?0@1^n_+N%gGx4EwxorPG&&J@h(d2WyCamXkglt+sAS>9PP4nRhgn|4p zd`|MQnrq22pctE&1hT!nXk50?#MGQ-X|3+1uR+n&)Ys9WXlZC_s%dDdY3Wk5v<$R0 z3^epyDLi_xzj_Fr%fj)5%ejGcy1I8@2xTsEl#rOjWX zHguto&F4}~0{MJ4lf;E!wkpNa%*>49z^8Ks^gquifn3T;cA!9HCem^&93vo5$fl64 z<#4@KDbzrJb!|OW3Xi9*siCc=si`Hh(hWNK-ACAq3z^6Z*@IT(m$s>ux-zdL;L_YbF zwAsHYzo^f@l3)2>$j=P)XHi5w|54)KwZBLQQb_TiSceJQ@LtB4#uLY}=yohK0Y*Kh+4z5=3d^2PanEcJ(q_h~GfeEMLX%d;&!E zIDXaYB|wxb9vV{U4a6<}>X2`bQJ$~WpCA7~Ih&i^6D@$qF3Vdv4QXbOr1jLPK&BSF6MWksux2S4>C#F4oGYZ{c`)*5BEUKs;=8sjSih;&J)U2Z9D5p6pOL zZLJN&Q@)$g$50?ze}(JE4h7;_Qq||aD?mK|x<8|9G7v8c52t6i0`XG*lE%r2NaN)u zD||<4=~&<$fpj%@*Sb`sNzNC(86r)+zNma3(sB34WT;~Q`o6p0_9MSL*;;LAzh{gt zulA8Zw7pA?)tL=MOWGJ2kA5JUJ~9=P@Ogu7KsdD(?KRf)&a-zw+zZWIZ}lGStEAel z)eJ<%2=Dw96ZFTsBjImz(N3ebR4+q+ZIS4FmQ)Qy@_F6@oy|a`ZtcEtawZV_ET+vW z4MBUAzE(2I2IA;)vI7vu_guTNc?S?D${q}_3_`muT~VXv2Sm<9cN#SZ@BKDBnuh&O zybXQS_XY3iL&KaT@xAg~YL5rrJ7@MKpxsX-yzWq|Kz=y~_e@8e9P_^6Xi$dZ%s%53 zBmu;c(At{C_khUSP0RRdiF9hw)Ev}%z<&oF6x=GEf!M2UvGC1xAa=Tx?h`7ZJ}oot zb5Q=6JLXsK3xSAuHtVCeEf67#(vKJI#&JH+tDS!YaS_2>n~#3I7+CmqqX6;Bb9Oq@ ziMV`eon>o^_wmGESO)f$j)cL#PC(@-X*XeoqK6BcLG{j5mVu<{(6Z>trbNL72 z_36dX=|Sk{W|e8Nh6#u>74IQ?5#J5n3!f^tpd1<&8S_d}uHs!6PHaUvc|BL!5wEvw ziZ);L2cpn#;`i-UK%8?NefZvIAWrEXx^ILy&TJjK|8ybRyVi=n?FA5LrXIQxg6#^U z{j04p}Y^S&2gH6ar$sc&we{T(k6a( z5&GlNwt~}@7!UOZr9JN#BUOw!zF7`whGWNd22x9b{QgJSj&a-MGUkzocC$y_j{Api z9rcjq@L@O}=UCgyUW|Kflimf`{%+z{_4GT4+q8`1`|1(5b58YlBff6b&wDk?3H_yP z7fwOH=jl3FzX(8ojtU)S{uGF`QwDX(%9wvTHZ_M7pkA+6^*K18okmPo*Zze3H{5pE zAdP;!v7dSrz`!+<`c(VkXGmkNUYtKN*MplM~qO>L=0fifSciJ25UE&+&eV{;SJN`#l5m zY`s#=3MP)%q{i>BL-|_VjVcQ_JnKHb;nV`0M+`d*+|X|?CrjMj~%gd{&q7{#6j-E%@Vs6NvYF)18&p4MTf(T=;$H^k04d`u8uM|HkisPQH3vQaxcS z=Htd&TZ21&aQ(L5_*5CrJI$drY3n;MFBiYw8;SX`)#SsTG)-Iw&ere9J&4q#qW*(4 zQa!D7osIbZ%cP=n*?2$v+ir85*IJ8<8}4sHYIpFoc`fqk-8^+E=Kq#GEo(yRkZzjf z&@c_@Cz+ceI6t<0F^uxScC8fi3Kr(k)}=`bRk%JIxNrH+U22Z&hL(GgwTtzTCR&WU zf%D%}Y43+?KcT!*!UAc`!;O>jI?w3hJl><%M1O|sw%-cDpYPzjLtJ{AwGrZmbY8nN zy$WI)73IA)?u6)|*g5O0njy+f(`6a$GDMKea;I(xe>)c!+A0v%^SwG||7QrDvtll; z@FAqXcw!ry3L&l&EVg;pK=A6V#I|>fL0B0U{nfn#1bwo@LvKj|pV&FQ+wmR*^=j*} zx~4&pp^q2NRKWXKtNJkGFa&)(6`(8}0RnPa8dLRPgxogY!~eAmr#byWt@eE zrFpO=k~!}~{6h%x9HV2yISxF=+6P@He?dTh*~*Nyap0%)P5U161o$dX>yAhU@EJ9i z!Yix=uk<;W_o&SQR&^@vt?(!?b4|urS}Fol`-<+kK21J;=LEwbCr0^8wd z(_`DM;FW0dOu{o1e3(lLA_WBaZpnMT`s#S#MjhDl;P4ONWi5;!XVC~juMA?I%AN;) zOVP!~-7z4X`zm!WPXU6L)_Hp^&xYWOg3b%xhap;X>n`&HDNUNe(0sVfDnq0qdnmVLRT8j*!>_#?jR8e0u>|qg%#?+FAnJbL5~Lhs!unkx*Y|6 zl-coL8~lJDsjnL=7!5)FL97auZxAFZ*Zd)9^`s3I35F1qRVRJo z&J_5!fq?OL7;DLP5bUE?83auP;p%f|3?|$MVS?}2_d8P{c%sy*l7n3kGDY%9-0olq z9sZfIWvc^(lgrV~S0O6urp3lPsSs`QN^7R45yaRyj~|Rbkw3{8RySF@EWzRXtRn4gtF6`%%1V-pMi3}sO9j7<4*P`d55V{9)rIZO z2f=SV3B;e^HzW1(?-6Ifmn^%KJ_6^riMM;pLg1_!;%R5IXps7B`A$C}YX&~W`(CjG zD}hsSy<}JNYw&%fYMA-{7WkDpv&`mv1OKFO<5y9c5TJCbKqxPOfU?mqJ8MgU+haId zv9cR@2(Edqg@ZT_UMt#&f-vX3X)=w6AxbwWZ$dd8qE{O(6P~GuSgUk1v-m&=58!O? zNDYFJQ+2h*HTED(-0tamAs2!=L(eQTPX_;#-2S_{3gE?z*KVko4a|O}SN;vRfsuCj zvE%hSK<5)OuXJs|^N`|%{YGIxyU-gzR2%~jgQ!#1)h4j+(EiKQlCr=RcPD0jJ^;?) zt1NzSCjpf!k#cZJ12|fE9iCaG0FIn5J3Mv10JVJL>=WHz!P)t`OW{xhaMe1erZe#z zxFuM`_1Vk@�Y<$V^~}<7PlEuEr4Stm81iJq&Y@D-V&>#Ly3tcHK<>2Sa{BbN@75rH3 z4jkEM;1^(0tMBs|e7m*;d|#*temck2L=0_!fWq~;S{8P|bE*qfTe1@POLcX-&rbm1 zme^12Icp$f*R7?c57ZzOc6OL|Q6bz>G$Fu-YwPv9m)Ao~N!A?7-cX2JsTj6mlo7;> z>&=9t;kVyKUV|9p6T6}mJ7A;!>WKM!RzXB-?Bh#1%@Ati-10ix6a-JVCY+-s0*`LY zT%VZ%{+*v~i+2@(pQMlJww^oSdu>RST5~gSJbNcLYwri2Y44_t@|1ffH%uvwWQa z_>Q<)GTW*R{G3-$J-dku{s-o}rcS>H0rq!WxRSRZ;0?KP>JEXowrA(qOalIw`>PyN za$tS^$#)dtJBYk;vdJZC6+~bADyI{o0`|gLl)LDY5xwz@8bbD?4;0Fh{kwKG<>(=x=DXs}|h{&vU=usmaU*T8Fhm z-CKX~$WH2h>2n9%#qr~|V5rte?=9deD!U!Qb#zy>OyoLnQcl*2lNkk6(=l@r`I0~t z&vQ;QRW6-BGXb3S)O|zt(!f>1&D^&8Ev!3BJAHk53b@xDnR(Pj7d-E-FiO=dADj=p z;qvh>Qwi{|t7x@}y$?LgIke~lQ-D8a;rkDd;z4-7HhFb^)L(u7`u8uM|HkisOuk}0 zo_FAI-$YX2biY3GHFu0|X z3-dkT7WHa`>Q_~83%jt&wBjtdjS#M=_?ZuGTI~EMB^$x*5Wj~@mj-vq>1A5S_Jg~@ zQO>XKr{FGLH@WQ_6OlPw9^8Uswz;J(1~=V(e#JyDxJ?`x86B_q5C2y8DI-T2gS&M) zt{~08{o`-zU;Fj~?c@2{Bc@ECuU+JRExi|*B@Rc+P0WFnMs@nh*$>|A_4YFg%zxN$EKMw(V zpYx`NT)(-9`MPYNuG9(o4$e&_&h_yN z!HH%W5A z>K(IIJ!FEbxDIsQCA{vtp%5RG~iePU$G2L?g3a}H`HFonXKfP>{ z0b8*>Yw|EFQNjo4o?N z-A)E!p{Zamt`i-^_;Vaf78vzF-PiRauRjM|$m)FfbXXVmWMo&@D)3l##z(K`26*Nc zSlN`l14jDQOO_84fhAriN2CjU@_ph4>%a($9jS>;?GQ1mG`K@|0fa}*og!=458+a2 zf&$gsgY|ZpKy+XLVOgJ!6K}O3?3CwD1MAHYR?|y-EiWFd&%$kFJ(3;9LijFG0SOU! zI2NpAKFB{p=KM2`p(I3%JP|}_9EXTeH*H?5WI}}M^4e3`Z4lwNpv%l|Dny9apAqLn zlg~P>hDbb|odT&4*~Zx%Hz^CE%>L~EZy0|rIZr7WHl(I5Tepb;8@Xj?L!W5C#s^Hf z+hZBv6&m@Lc53Tj{m!Uv!JV20px1BwagjL}JnyJ@WKJ6io{E}ZxDU*MCSJeMmL=vS z-BSYEifdl{pIJcDRaU(!PzR5XjXSIZeu2l`Rfpw<&H|76)6+5Zz(aH_HwI|K>c|}l zpefN7D-+#TNk|@!-0X--|3$VfuUKx z`sC^xgZYgf<;OmmBL(ze>9>T*akG8Gq!TM8-b6?T6f))5r`*U7AkaygZ9AcL^NW!|Rq}gh={K2?z66ZJS)EQ*u z0uH9GM_!caf!z&@iDpy|%)=#cZ zQ}FBsYw>xfHJ&z&KbHd5BZdvTwc#09iE(J<^rZRZh$CQS9YSl&&jTy*`L&g&ky9A+ z09aj*I~lM>2dvLjo|R6U4>m5HPP43~z*cNm`|bVLU(SjGNAh!xgBjqIwdi84rWCMd z*IYQ_;s*5R6SV{0?FCQr>jjfuaL;`n6n3)>TnpcH>X*#~=Yx$)44!li=1C`$x6G#& zeuH^~DmrGL3{){L9CKRF_O>ko$IUjyR|F2=$afyCvHcx5iu1i=zC8EE=u_Yrr${0k z9K)~Smj=%V>tILm{*@zsHISVb`wt&7OHzw~I`*nzwv8cB!}nC$7h8anp2LDUH-CVu z$SoG&k>VLGaOwbt{1P|swim$qsINY=#}N23y^Ex>M?%n6S-Gc68-RCH^Z^0z$YbUo zy9f6Xcw#&7P{IidC{$}%< z5*rY#+lf0pJs@~iRWN=(6FfWo&kb9V3!dWo$>T{|Q+d)Za2MB^>j=yH)Xqv+C(f%b zSL9B1e7p?KYvw8ThgE{}?8OBi&VK`^*HLy$b@jkWT=zO{mRdoh%o@azQ^O_Kv}j9k zUT{F;hU^D$Ho1Ll(abt<#?xD=O%dQcF)L#6f_QNH6(U`#kOfX{3Jt!^%fU%pj}G)7 zy#H=$+5BD!oW=9%K>g@^MTfI{iVe8DWA3PukA`(gSwm8@^}wy$@$Sx#r~c~u*S~-9 z{5O98WAYW#OV0LzYc4PiB&BPjrUS!3r6TOaMPP`}VHlfBByN480OO5;e8zCyf6jf# zuNp^R21a62+r{B6z`(Bp7k;$`hIpTlsWi&YbmQ^m3BV+GfBkj< wYsj)v&kr%c%wPA>h}Hru<=;U`36FuvBR7^kft4PpOGHzFRn(`f@m}-40I=l@NdN!< literal 0 HcmV?d00001 diff --git a/parm/ioda/bufr2ioda/amsua_n19.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_n19.ACCoeff.nc new file mode 100644 index 0000000000000000000000000000000000000000..bc36197dececc20d8fdca2c0ce4acf78daa42c0a GIT binary patch literal 12460 zcmeHNc{G*px0j((p?Og1D3uBsqNr@gkR(IMP&ke=a|UN{Op%D@qDceIqFHlDq@ofT zA|*6wuD29T)ZH&f-+R}(zq@{a+<$&+b=G>%KJPQ`;j^D-@8>yo_Vaovb$|LHQO2jY zkT%cFTiR7P7Qg!mX?9$?P{`tUmuX?y0J0pPp%D_Ej71ZOnBjaDjmwkDM3N{hC&zKs zb7ERisCITD7Mo2IvSfDl^Jq?PMr=!CiiwHkj2RSDV-piYV-rJD3yP_!m6@@X@ysQZ zFnT!ONKO|raXy)iP(-I2aYS;89*(G_G@Kz}(Pcc5kjA9ryx58+;|W+@b>y5n2R)PE4|F zY=!k|UOXAUTZ}HD^@%k&xoZ5=pz%L~4$Sa|lE-7KJn|PslN#c#HT( zW-|>aVPQrl#%6{lCZ=6_>W|-~oir5~P&h(YT@P+ZUm)A4^F=}qO-NV3u}f}6Kgxga z*PqFfG9z@OV#h{GX>lIZ+z?a zSmF-~qZAm#i70X&E23+Pf1CH-*t{2f?Cyl$_^vq+$uj(n{7XsXla-{-{!RY7`ur>Y zC;bcl86rND($(`HA^u(ccga99srV>l7`h(c`%!6O3P0|u{deW)Pu1_~M~d>l+QR5x z$Rh1omMA?#{J+UGJ{@@>?Lf+*f)9x|a?K=l&cT>qRGK|2e?jrR+*>l}b6?d@8z%K+*A|T&;(YS$H67URoa&ne>xThxW`}aop?yGn zhzousoCd^4j@r%bjX<<6y1M0T77(9DcDTUx_DJ2gJ!EK^MMv0&!OGc3Vph>MMAC!C>UG@PomQ7Ql9fDgG)J*gsTa{iFcL6FaA0 zz;Wtwv^ozVzx7=Clcp0$^K2ey>LFzZD!)Oy)-{<;%J`0S^wi0pE@HnqjdhQa|C(h( zt$ys>;o${%EIdl#RoBjRImfP_iH#h=XHecdo9%anj3TsWQYzwbOGq#KXnx z0pFh`<2+uf%)xi@{piH{`)J<_85aWYSYtaq%iRVz{uvVX0YK!)vc$KAk}s} z@)YsWP{OP!oQ`y>*1JYcr1`HRe_X+Fp1rt!BMrxmG5%KKj`-|1H|*ejoVRSxT}C|O zi|0pse;9F5utF*sd-pu!W2Eshd)7vj^A9T2CT0U>?es^{tm-C(f7Veai~xuR1g{!NmpT zoSJv#1oo@Qez<(jSF}Tx#<+uKi2HLxb}%f_KBC<_UA56}YPAN#n-OojGnUs+2I91E z<}e$?`SJUAX|DF@xAA5C=vc&IUCe=*XotMwW(}$d`ft)Ajam3!awhbfeD6PgRX#l+ zKOJ#)ci79DRp_UcbSfhP>#KBr-*C)8JB(|6z7f}p9{tn=RqBO}M>zshlPaEI{oco_ z7u&J@VR!6_)bW$CAL6J=`ZJn_`mZDnU4m3ZEYm~(sL(h%;3SE&q~*r+o5%;b{CS|9 z!*9=fyy>5Hy>RWxF1w0C7b3&V%4Sh>wxU%?W2vUmAsKjQ2>_ZJD&fcCr;`#yH|Qnd+8I#N~;12NqBH zg;e|P+^R9?&*L^)6``D2ExJBj)LSNx?14CJ&YiUz<=c}f8Tt|FZsubz9{Rx!%9z^k z=|F6)xUN%m0EkVRDVJjqABkh$v@R+JBJ#vU&JMJP;OOQTx~4!druJr(qM!8WBQMBx ztXLK8%Ad5ev;g{spHq@i?_(#f{8Ebcn8cut_=ffx5^%Im z8SOcGH~Tx{tJdG+rQv?WY1Z)-HAfN0ef3gL6(KczZv9{h`e$umXvui=+c*99e?fU3 z%{)m(3q5z!tp3Cix zMu@Z57YoO3M0>ARcMVvIxE_7!+EnDXur<)T9Q`z(T)w8GU#C%Qw`Kj6_pg2b;`y)t z{^zV$A4cq(e!db2!m=-I!C;KzuPxMGi~B-r#0r~68_Wl`AFZmi1)|-of{}^qeEW&8 z7p<>>_-dAY@Y@cg%P0SG#yIfn>18d3xKDk}{|>EzNUsbY`vT*dujwnNXR0G5!{894 zNs`_AT1YW1a>jkfjrp%CeI%gr}$AYHXCddwuGW74!@_Tao-ffv%H zhly(vAlg3cJZw~h)NN+?r6HK7toO3GgmJ;g&#T8rR-ru7qN^RaKljM@UMpC-c`wk) z^_~RPuZ95T)>_THli)v0eOdgJt%WId1mEZd^S=v+gHOPa`!0TN z;QjXA-O3FM!TaUGL9b|);FGYGc>{0eM1cgL7Qfa;hVriXSw(8Oh_&cl}j{pOp( zj)!M}aY|NhBHs;c{jw^(1;N0D)EDzUc>`~gj&<$i76^fP?00)6KuGzMy~Ww9K)5j| zUh4A#!b-mGJO6$INOtYNzxZ+)$mXvPDr-Fh;i=_gVq5w`1ZGF;vH_wzefimYOCiSU z!)qT$Er_`W7&uu%Orb_8x48hkhrRu=DfS|G?0Y9O?WGE?=O~8hpXS0s--%0Amezvv z+mVBW1un4Q<$xh(xuLMYT_bRV*C1F>pq;JTx)Yp}g?2}yE`v*K;r!)go579DO1;xy z(PUB1W|f2BF)ij~d7mWk)ZR!U0lc(%Gpl~FU~z6{_U>(Cz%M!Xn(CboutdDurK0d1 zEFnkLGKK*0RLxR53Ir`mZgBsi2@JO8qUt5bfO9V+<}hzPgvL)eTw$IFg0Aq$8wC5= z!_T`;15v=QOZ5|MAxsfBi38dfV?+RA*V*Of*}f2$wf6IKZW%1Y;9_>U1w>OCOGbII zAj0V5Nl%tCi0=&44C$xYg&-rCR9RvOWl0#;kM8qy%s=`a0dd z8h9)BJs)_%1VTDgZ(dHn2chRu`GqGSE z)W&Dw$q;X0xBBfJJ|wn;TUi%Phvd1}lTR3y!77Dcukom!@pjh;h@OAJ15W}VD)5QL z%P|ij;)&_`z)!zG-j89tNa+wrLRfb>r;C6eDEivGPZN019dB|PmA=ZaCjk3n?+uso zN`ZBozi*CnKVVI&KiwE$4b0|s!z8`~fVt^Gr^VOjz;x|Opnz%Uq{%KD2u!mr$LzG6 zfbpa&!2zcH)S4SFegkXA2y!I=&RMPX%+Axm+n`E&GH5b{l4-eYE%29ETYi;n0l~W# zao_lQ2y6K0ZC%+8vV$dd4tGC5)KK95bS!3MZhALhYKyhYA{Pf6#<(}OCp+pEmYdL zdsZE=mz~UVtnUr%)^YsfRY|~si3U0VHUAXoP;s%vuq8P(h?{Gc{Std}Oy?-_&8O-`#naU4SOd)+c@XaJtZ2$Cb< zS~@y~oX`Pw`=Y5^e=7mIj!fvE0b4N+Yy9EQZPSZ^>Diq(2cO!tzG>eKU?|Qbn7Ej_ z-sS>J+b&Y?-W_1xjEZjycmXW61;x8RF@Uvj2d#JeR^S|t_BbAH4ZP7ee3=er?&^VJT1_QFj64 z(8$_vzXrm;r_ah+Z~?+gte$%5Y=UsidUP*70oj4vnzHMgKsL>Iy5yWENPmqAZ!Kej z%trtD&$N7ytz2aKExHP1FF&N(o>hVHDZiAQmoEYNQp%;?{Amzj(4Ch@YRvh~yt@IS zB<0K6zC4HM@bdl_-g6*^p0D=f?R8i--G&yr5Bcotny6hp1mYC?%8L5T`4f_CAo0tQ zwb?99Nc#PvcKt>IRuN0KyRM7-!?#S~Z_-7>(w!n~L40Y#O@F=XAUNkh9?U}Mkl_KA z+kXOIu}=xs3+M(R#lXMvs!#p}J>U;GtduiFA40ZFnLBV=Dex-#&_vY~;H`Zd|L#W! z@Z8PDTE!j(HY;QB?Z#$cS`X~qC;k^OUJZ389;E}5{B}74%*dPE+|B}E+0S@pu{#4e zvy)!+e=h@WXu9#jiWk6(^LCse{sy5l4lRsPt^GHlC#3|n-#Q6km-}gQ+bcJ)O-!+^FFy1SmE3ZkHQ*vpYdG3h z7qWrc>6=tDWER+BGB|Ykkbl~(`+Vhy&Tg>v+jGC849Mdj zvy9bNfSg-QpY^)~SM@KhKzO z8NzjUt&Gvb{gzB))}060kilez1X6__rBj`jXI5ig*YC%4wJdcIPxP+Jc-I19ob^hc zPwPPRifdM)G!;af`Ee(g-ULz5h_pLaN+4{TU^wlJIS4~?&&;iL1%X@oxTO0I5SrCt z`nDwvczs(|+>ML^_5!~m)`hviROA&*S$F>46X!WSOy8-$WHUI8*J`s_69Ds7A0%#- zrGcZ}YsRDbq2Q3+l=Gu`Iyek!?(mAL1^YwQX9^o!z)rEB+u`Mxbl}cEcChWS+$&0uO(1^2%ypMsEwSpLv5fZO#pFkgvHtvVi?({sjj=&d7Ok z9u{tHEI41k4P3j->j&%wsvod1nI8cRzVC;Vd!9q^(u)`J2d4o2Y->O1S0|v);j*<~4gk8z zyZ+wcz7Ry3`Oy)e#Y%5Yj0}Q6?!FqQD=84L;!|(VcTMo0s%)yxSq=WqDY^@47~s#$ z+nWE}5&SVJH)u0}0K48T<*qdlaAUypH_$-9q`5&xCUpm&gdP3kk9)UzN0pRz<9FOtOR`s8**f- z@nTaDbJ#YAb1{$X#ayX5LK|d?I7VjYYR1d02kG~hm-=PA2I;$bn)Urgg0wXSGm1qZ z?bPTUS)>Ls-+8ZEGpHcj{M#9CQbG21NOMjt1;TxGr#MKnA^gZ&itNoskSp>8dDtky zCbz91SDb%Dkk?49KOo}wZg11smm!iqu6A{R2}Fv@Kg{T$L*&g5pYSRNqD0w7tEz88 z^xCcqafmq+xgp2L8J1nXt6#d84Y3%_t}fAlIHzgF9Xu&4=LiptU$X<^!wrul%d=nw zdF??>f&>Qxzn^0~ zwnLyIpQafW=e!v?6N36#TDN>?grHgeoaSd!At>0T`qPmP2+B%G;qC~7pm!HtnvoP^;7Bc72pmb7mQK} z3A*QPr2ZTDj)9Ee%vZoUuOI3+R0!;Q*JgWenFy?Q`5RmPoxlp9d1oC@0A`3bd4dBB z#eGRne#7YhxT$(|1q5IG)%WDjX%KvQdP$o8VFzp0l~+(744E-2ri`hP>;ecM(`E zr&{+*jsv^;s6TBP2RMa?N_3JZ0C$t^uY^T$z~k7TxfoFeAyk^f<8x0Slsp|8k_Upt zFD?gf`xcvmG8FFKtBZcgR+|QFiR;>Z@5KF7 zL!PJruZ}!`8Szg(kx}NeqI4MWuB}r&78D90 zVw+uhDnnO+7&e?+qdw)&`XIi%IN9C6wC{Jr375WXka+d>cx@|jVuD_?I0`J0Qgc9CtFGLnR1LIKH*@;z z(g)h*fY(LicR^6E+H1qMynvtq9y$-z@ZIpS#%Qq|g1X>00fL@;R%=ar4?XkIziPZ{ zulW7PpBUW9#wB7NFckYFi_>tLKh^>`CmSo4Rhk1&r)j3|(x-pr{cB(H!2K_t|N8HL z#Cqj8XzQ{clMcYVgzxrxV*nf!=fn=Pt1tr10{e841rhHJfAWHonTIHlHo|xexxl zCw2XS32g8buk$J$Y-|djPZm^16zN`C?qZ(uZSj{1 ac.n_fovs)] = [INVALID] return t diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index 29e9ed83d..9f5440401 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -5,23 +5,22 @@ import os from combine_base import Bufr2IodaBase, CPP from wxflow import Logger -from antcorr_application import ACCoeff, apply_ant_corr, remove_ant_corr +from antcorr_application import ACCoeff, apply_ant_corr, remove_ant_corr, R1000, R1000000 from utils import timing_decorator, nc_merge logger = Logger(os.path.basename(__file__), level='INFO') -R1000 = 1000.0 -R1000000 = 1000000.0 - BACKEND = CPP -AMSUA_TYPE_CHANGE_DATETIME = "2000120000" +AMSUA_TYPE_CHANGE_DATETIME = "2023120000" -YAML_NORMAL = True # current as normal +BAMUA = '1BAMUA' +ESAMUA = 'ESAMUA' +YAML_NORMAL = True # current as normal -class Bufr2IodaAmusa(Bufr2IodaBase): +class Bufr2IodaAmusa(Bufr2IodaBase): def __init__(self, yaml_order, *args, **kwargs): self.yaml_order = yaml_order super().__init__(*args, **kwargs) @@ -34,7 +33,6 @@ def get_yaml_file(self): class Bufr2IodaAmusaChange(Bufr2IodaAmusa): - def get_yaml_file(self): if self.yaml_order: return self.config['yaml_file'][1] @@ -59,7 +57,9 @@ def re_map_variable(self, container): ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) else: ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) + logger.info(f'ta before correction1: {ta[:100, :]}') tb = self.apply_corr(sat_id, ta, ifov) + logger.info(f'tb after correction1: {tb[:100, :]}') self.replace_container_variable(container, 'ObsValue', 'brightnessTemperature', tb, sat_id) def apply_corr(self, sat_id, ta, ifov): @@ -70,12 +70,16 @@ def apply_corr(self, sat_id, ta, ifov): # Convert antenna temperature to brightness temperature ifov = ifov.astype(int) - 1 for i in range(ta.shape[1]): + logger.info(f'inside loop for allpy ta to tb: i = {i}') x = ta[:, i] + # logger.info(f'ta before correction: {x[:100]}') if self.yaml_order: - apply_ant_corr(i, ac, ifov, x) + x = apply_ant_corr(i, ac, ifov, x) else: - remove_ant_corr(i, ac, ifov, x) - x[x > R1000] = R1000000 + x = remove_ant_corr(i, ac, ifov, x) + # logger.info(f'ta after correction: {x[:100]}') + x[x >= R1000] = R1000000 + ta[:, i] = x else: pass # TODO after know how to set llll return ta @@ -118,10 +122,12 @@ def merge(amsua_files, splits): yaml_order = YAML_NORMAL else: yaml_order = not YAML_NORMAL - - for sat_type in ['a', 'e']: + logger.info(f'yaml order is {yaml_order}') + BAMUA = '1BAMUA' + ESAMUA = 'ESAMUA' + for sat_type in [BAMUA, ESAMUA]: logger.info(f'Processing sat type: {sat_type}') - if sat_type == 'a': + if sat_type == BAMUA: convert = Bufr2IodaAmusa(yaml_order, args.config, backend=BACKEND) else: convert = Bufr2IodaAmusaChange(yaml_order, args.config, backend=BACKEND) From bd04d95889ea0b0230eff739e06727da4ce93851 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Mon, 22 Jan 2024 15:39:20 -0600 Subject: [PATCH 55/58] code norm check --- ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 4 ++-- ush/ioda/bufr2ioda/run_bufr2ioda.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index 9f5440401..4d1e61adc 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -15,7 +15,7 @@ AMSUA_TYPE_CHANGE_DATETIME = "2023120000" BAMUA = '1BAMUA' -ESAMUA = 'ESAMUA' +ESAMUA = 'ESAMUA' YAML_NORMAL = True # current as normal @@ -87,7 +87,7 @@ def apply_corr(self, sat_id, ta, ifov): @timing_decorator def merge(amsua_files, splits): - ioda_files = [(f'amsua.{x}_ta.tm00.ncc', f'esamua.{x}.tm00.ncc', f'amsua_{x}.tm00.nc') for x in splits] + ioda_files = [(f'amsua.{x}_ta.tm00.ncc', f'esamua.{x}.tm00.ncc', f'amsua_{x}.tm00.nc') for x in splits] logger.info(f'Ioda files: {ioda_files}') file1 = [f for f in amsua_files[0].values()] file2 = [f for f in amsua_files[1].values()] diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 976192ce6..7e24d26a0 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -99,7 +99,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): BUFR_yaml_files = [os.path.basename(f) for f in BUFR_yaml_files] BUFR_yaml = [f.replace('bufr2ioda_', '').replace('.yaml', '') for f in BUFR_yaml_files] logger.info(f'All obs type processed by yaml: {BUFR_yaml}') - BUFR_yaml = ['ncep_1bamua_ta', 'ncep_esamua'] + BUFR_yaml = ['ncep_1bamua_ta', 'ncep_esamua'] for obtype in BUFR_yaml: logger.info(f"Convert {obtype}...") From 086cff31a6d8f4935e8fbcc1d700dfed1708322c Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Tue, 23 Jan 2024 08:06:20 -0600 Subject: [PATCH 56/58] for ctest --- prototypes/configs/cp0.sh | 4 ++-- prototypes/gen_prototype.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/prototypes/configs/cp0.sh b/prototypes/configs/cp0.sh index 257ed5f7a..e91d22240 100644 --- a/prototypes/configs/cp0.sh +++ b/prototypes/configs/cp0.sh @@ -14,8 +14,8 @@ edate=2021080118 app="ATM" starttype="cold" gfscyc=0 -resdet=96 -resens=96 +resdetatmos=96 +resensatmos=96 nens=0 # config.* options DO_JEDIATMVAR="YES" diff --git a/prototypes/gen_prototype.sh b/prototypes/gen_prototype.sh index d58492e4b..ee531e922 100755 --- a/prototypes/gen_prototype.sh +++ b/prototypes/gen_prototype.sh @@ -89,8 +89,8 @@ cd $GWDIR/global-workflow/workflow --app $app \ --start $starttype \ --gfs_cyc $gfscyc \ - --resdet $resdet \ - --resens $resens \ + --resdetatmos $resdetatmos \ + --resensatmos $resensatmos \ --nens $nens \ --pslot $PSLOT \ --configdir $GWDIR/global-workflow/parm/config/gfs \ From c49d17dc48de37db4d728b13b6563f2a94721b3f Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Tue, 19 Mar 2024 19:04:31 -0500 Subject: [PATCH 57/58] make it working after merging develop --- parm/atm/obs/config/amsua_n19.yaml.j2 | 585 +++++++++++++------ parm/atm/obs/lists/gdas_prototype_3d.yaml.j2 | 1 + ush/ioda/bufr2ioda/run_bufr2ioda.py | 2 +- 3 files changed, 395 insertions(+), 193 deletions(-) diff --git a/parm/atm/obs/config/amsua_n19.yaml.j2 b/parm/atm/obs/config/amsua_n19.yaml.j2 index 06ec22687..25f0a914a 100644 --- a/parm/atm/obs/config/amsua_n19.yaml.j2 +++ b/parm/atm/obs/config/amsua_n19.yaml.j2 @@ -3,7 +3,7 @@ obsdatain: engine: type: H5File - obsfile: '{{ DATA }}/obs/{{ OPREFIX }}amsua_n19.{{ current_cycle | to_YMDH }}.nc' + obsfile: '{{ DATA }}/obs/{{ OPREFIX }}amsua_n19.tm00.nc' obsdataout: engine: type: H5File @@ -12,15 +12,18 @@ max pool size: 1 simulated variables: [brightnessTemperature] channels: &amsua_n19_channels 1-15 + obs operator: name: CRTM Absorbers: [H2O,O3] Clouds: [Water, Ice] Cloud_Fraction: 1.0 + Cloud_Seeding: true obs options: - Sensor_ID: amsua_n19 + Sensor_ID: &Sensor_ID amsua_n19 EndianType: little_endian CoefficientPath: '{{ DATA }}/crtm/' + obs bias: input file: '{{ DATA }}/obs/{{ GPREFIX }}amsua_n19.satbias.nc' output file: '{{ DATA }}/bc/{{ APREFIX }}amsua_n19.satbias.nc' @@ -51,24 +54,180 @@ ratio: 1.1 ratio for small dataset: 2.0 output file: '{{ DATA }}/bc/{{ APREFIX }}amsua_n19.satbias_cov.nc' - obs filters: - - filter: BlackList + + obs pre filters: + - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature channels: *amsua_n19_channels - action: - name: assign error - error function: + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false + + obs post filters: + # Step 0-B: Calculate Derived Variables + # Calculate CLW retrieved from observation + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/surfaceParam + type: int + function: + name: IntObsFunction/Conditional + options: + defaultvalue: 0 + firstmatchingcase: true + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + value: 100 + - where: + - variable: + name: GeoVaLs/land_area_fraction + minvalue: 0.99 + value: 85 + - where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.99 + value: 90 + - where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.99 + value: 85 + + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/thinningCriteria + type: int + function: + name: IntObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/surfaceParam + coefs: [1] + + # Calculate CLW retrieved from background + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + + # Calculate symmetric retrieved CLW + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + + - filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + + # Calculate scattering index from observation + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + + # Calculate CLW obs/bkg match index + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *amsua_n19_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + + # Calculate symmetric observation error + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_n19_channels + type: float + function: name: ObsFunction/ObsErrorModelRamp channels: *amsua_n19_channels options: channels: *amsua_n19_channels xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] + name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, 0.100, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.030] @@ -77,39 +236,109 @@ 0.000, 0.000, 0.000, 0.000, 0.200] err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] + 0.400, 0.550, 0.800, 4.000, 3.500] err1: [20.000, 18.000, 12.000, 3.000, 0.500, 0.300, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 18.000] - # CLW Retrieval Check + 0.400, 0.550, 0.800, 4.000, 18.000] + + # Calculate Innovation + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *amsua_n19_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *amsua_n19_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *amsua_n19_channels + - name: HofX/brightnessTemperature + channels: *amsua_n19_channels + coefs: [1, -1] + + # Step 0-C: Assign Initial All-Sky Observation Error + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n19_channels + + # Step 1: Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + + # Step 2: Data Thinning + - filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + priority_variable: DerivedMetaData/thinningCriteria + # round_horizontal_bin_count_to_nearest: true + # partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + + # Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check filter variables: - name: brightnessTemperature channels: 1-6, 15 test variables: - - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] + - name: DerivedMetaData/CLWRetFromObs maxvalue: 999.0 - action: - name: reject - # CLW Retrieval Check + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + + # Step 3B: CLW Retrieval Check (background_based) - filter: Bounds Check filter variables: - name: brightnessTemperature channels: 1-6, 15 test variables: - - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] + - name: DerivedMetaData/CLWRetFromBkg maxvalue: 999.0 - action: - name: reject - # Hydrometeor Check (cloud/precipitation affected chanels) + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + + # Step 4: Window Channel Sanity Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + + # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -121,217 +350,181 @@ channels: *amsua_n19_channels obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] + 0.400, 0.550, 0.800, 4.000, 3.500] clwret_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] + name: DerivedMetaData/CLWRetFromObs obserr_function: - name: ObsFunction/ObsErrorModelRamp + name: DerivedMetaData/InitialObsError channels: *amsua_n19_channels - options: - channels: *amsua_n19_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, - 0.100, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.030] - x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, - 1.500, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.200] - err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, - 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] - err1: [20.000, 18.000, 12.000, 3.000, 0.500, - 0.300, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 18.000] maxvalue: 0.0 - action: - name: reject - # Topography check - - filter: BlackList - filter variables: - - name: brightnessTemperature + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + + # Step 6: Observation Error Inflation based on Topography Check + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_n19_channels - action: - name: inflate error - inflation variable: + type: float + function: name: ObsFunction/ObsErrorFactorTopoRad channels: *amsua_n19_channels options: - sensor: amsua_n19 + sensor: *Sensor_ID channels: *amsua_n19_channels - # Transmittnace Top Check - - filter: BlackList + + - filter: Perform Action filter variables: - name: brightnessTemperature channels: *amsua_n19_channels action: name: inflate error inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n19_channels + + # Step 7: Obs Error Inflation based on TOA Transmittancec Check + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n19_channels + type: float + function: name: ObsFunction/ObsErrorFactorTransmitTopRad channels: *amsua_n19_channels options: - sensor: amsua_n19 channels: *amsua_n19_channels - # Surface Jacobian check - - filter: BlackList + + - filter: Perform Action filter variables: - name: brightnessTemperature channels: *amsua_n19_channels action: name: inflate error inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n19_channels + + # Step 8: Observation Error Inflation based on Surface Jacobian Check + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_n19_channels + type: float + function: name: ObsFunction/ObsErrorFactorSurfJacobianRad channels: *amsua_n19_channels options: - sensor: amsua_n19 + sensor: *Sensor_ID channels: *amsua_n19_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - # Situation dependent Check - - filter: BlackList + + - filter: Perform Action filter variables: - name: brightnessTemperature channels: *amsua_n19_channels action: name: inflate error inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_n19_channels + + # Step 9: Situation Dependent Check + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_n19_channels + type: float + function: name: ObsFunction/ObsErrorFactorSituDependMW channels: *amsua_n19_channels options: - sensor: amsua_n19 + sensor: *Sensor_ID channels: *amsua_n19_channels - clwobs_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] clwbkg_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - bias_application: HofX + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs scatobs_function: - name: ObsFunction/SCATRetMW - options: - scatret_ch238: 1 - scatret_ch314: 2 - scatret_ch890: 15 - scatret_types: [ObsValue] - bias_application: HofX + name: DerivedMetaData/SIRetFromObs clwmatchidx_function: - name: ObsFunction/CLWMatchIndexMW + name: DerivedMetaData/CLWMatchIndex channels: *amsua_n19_channels - options: - channels: *amsua_n19_channels - clwobs_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - clwbkg_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - bias_application: HofX - clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, - 0.100, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.030] obserr_function: - name: ObsFunction/ObsErrorModelRamp + name: DerivedMetaData/InitialObsError channels: *amsua_n19_channels - options: - channels: *amsua_n19_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, - 0.100, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.030] - x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, - 1.500, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.200] - err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, - 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] - err1: [20.000, 18.000, 12.000, 3.000, 0.500, - 0.300, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 18.000] obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] - # Gross check - - filter: Background Check + 0.400, 0.550, 0.800, 4.000, 3.500] + + - filter: Perform Action filter variables: - name: brightnessTemperature channels: *amsua_n19_channels - function absolute threshold: - - name: ObsFunction/ObsErrorBoundMW + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_n19_channels + + # Step 10: Gross Check + # Remove data if abs(Obs-HofX) > absolute threhold + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound channels: *amsua_n19_channels - options: - sensor: amsua_n19 + type: float + function: + name: ObsFunction/ObsErrorBoundMW channels: *amsua_n19_channels - obserr_bound_latitude: - name: ObsFunction/ObsErrorFactorLatRad - options: - latitude_parameters: [25.0, 0.25, 0.04, 3.0] - obserr_bound_transmittop: - name: ObsFunction/ObsErrorFactorTransmitTopRad + options: + sensor: *Sensor_ID channels: *amsua_n19_channels - options: + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_n19_channels - obserr_bound_topo: - name: ObsFunction/ObsErrorFactorTopoRad - channels: *amsua_n19_channels - options: + options: + channels: *amsua_n19_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_n19_channels - sensor: amsua_n19 - obserr_function: - name: ObsFunction/ObsErrorModelRamp - channels: *amsua_n19_channels - options: + obserr_function: + name: DerivedMetaData/InitialObsError channels: *amsua_n19_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - bias_application: HofX - x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, - 0.100, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.030] - x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, - 1.500, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.200] - err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, - 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] - err1: [20.000, 18.000, 12.000, 3.000, 0.500, - 0.300, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 18.000] - obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, - 2.5, 3.5, 4.5, 4.5, 4.5] - action: - name: reject - # Inter-channel check + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_n19_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + + # Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -341,14 +534,19 @@ channels: *amsua_n19_channels options: channels: *amsua_n19_channels - sensor: amsua_n19 + use passive_bc: true + sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, - 1, 1, 1, -1, 1 ] + 1, 1, 1, 1, 1 ] maxvalue: 1.0e-12 - action: - name: reject - # Useflag check + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + + # Step 12: Useflag Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -361,7 +559,10 @@ channels: *amsua_n19_channels use_flag: [ 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, - 1, 1, 1, -1, 1 ] + 1, 1, 1, 1, 1 ] minvalue: 1.0e-12 - action: - name: reject + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml.j2 b/parm/atm/obs/lists/gdas_prototype_3d.yaml.j2 index a8bc25019..8e9c7ba7c 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml.j2 +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml.j2 @@ -1,6 +1,7 @@ observers: {% filter indent(width=2) %} {% include 'atm/obs/config/satwind_abi_goes-16.yaml.j2' %} +{% include 'atm/obs/config/amsua_n19.yaml.j2' %} {% include 'atm/obs/config/scatwind_ascat_metop-b.yaml.j2' %} {% include 'atm/obs/config/conv_ps.yaml.j2' %} {% include 'atm/obs/config/gnssro.yaml.j2' %} diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 7e24d26a0..3029a0ed2 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -57,7 +57,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # copy necessary fix files to runtime directory fix_files = ["atms_beamwidth.txt", - "amsua_metop-c_v2.ACCoeff.nc" # TODO need to find a place for this .nc file. + "amsua_n19.ACCoeff.nc" # TODO need to find a place for this .nc file. ] for fix_file in fix_files: shutil.copy(os.path.join(config_template_dir, fix_file), os.path.join(DATA, fix_file)) From ba8a1fa29ab39c95908b728a056698066dcd554c Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Thu, 28 Mar 2024 22:24:13 -0500 Subject: [PATCH 58/58] some clean and log --- .../bufr2ioda/bufr2ioda_combine_ncep_amsua.py | 18 +++++++++++------- ush/ioda/bufr2ioda/combine_base.py | 13 +------------ ush/ioda/bufr2ioda/utils.py | 1 - 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index 4d1e61adc..7e14f740d 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -57,9 +57,9 @@ def re_map_variable(self, container): ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) else: ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) - logger.info(f'ta before correction1: {ta[:100, :]}') + logger.debug(f'ta before correction1: {ta[:100, :]}') tb = self.apply_corr(sat_id, ta, ifov) - logger.info(f'tb after correction1: {tb[:100, :]}') + logger.debug(f'tb after correction1: {tb[:100, :]}') self.replace_container_variable(container, 'ObsValue', 'brightnessTemperature', tb, sat_id) def apply_corr(self, sat_id, ta, ifov): @@ -70,7 +70,7 @@ def apply_corr(self, sat_id, ta, ifov): # Convert antenna temperature to brightness temperature ifov = ifov.astype(int) - 1 for i in range(ta.shape[1]): - logger.info(f'inside loop for allpy ta to tb: i = {i}') + logger.debug(f'inside loop for allpy ta to tb: i = {i}') x = ta[:, i] # logger.info(f'ta before correction: {x[:100]}') if self.yaml_order: @@ -114,8 +114,12 @@ def merge(amsua_files, splits): amsua_files = [] splits = set() json_file_name = args.config - with open(json_file_name, "r") as json_file: - config = json.load(json_file) + try: + with open(json_file_name, "r") as json_file: + config = json.load(json_file) + except FileNotFoundError as e: + logger.info(f'Json file not existed exception: {json_file_name} with error msg: {e}') + exit cycle_datetime = config["cycle_datetime"] if cycle_datetime >= AMSUA_TYPE_CHANGE_DATETIME: @@ -128,9 +132,9 @@ def merge(amsua_files, splits): for sat_type in [BAMUA, ESAMUA]: logger.info(f'Processing sat type: {sat_type}') if sat_type == BAMUA: - convert = Bufr2IodaAmusa(yaml_order, args.config, backend=BACKEND) + convert = Bufr2IodaAmusa(yaml_order, config, backend=BACKEND) else: - convert = Bufr2IodaAmusaChange(yaml_order, args.config, backend=BACKEND) + convert = Bufr2IodaAmusaChange(yaml_order, config, backend=BACKEND) convert.execute() amsua_files.append(convert.split_files) diff --git a/ush/ioda/bufr2ioda/combine_base.py b/ush/ioda/bufr2ioda/combine_base.py index f2d5987a3..52be538ed 100644 --- a/ush/ioda/bufr2ioda/combine_base.py +++ b/ush/ioda/bufr2ioda/combine_base.py @@ -19,9 +19,7 @@ class Bufr2IodaBase: - def __init__(self, file_name, backend=PYTHON): - with open(file_name, "r") as json_file: - config = json.load(json_file) + def __init__(self, config, backend=PYTHON): self.config = config yaml_config = self.get_yaml_config() self.yaml_config = yaml_config @@ -36,15 +34,6 @@ def __init__(self, file_name, backend=PYTHON): self.yaml_path = None self.backend = backend - def check_yaml_config(yaml_config): - # TODO make some consistency check? - pass - - def get_attr(self, config): - # TODO add more attr into the ioda file from config - obs_attr = {} - return obs_attr - def get_container(self): container = {} if self.backend == PYTHON: diff --git a/ush/ioda/bufr2ioda/utils.py b/ush/ioda/bufr2ioda/utils.py index 3fcac6635..76c3b7ba0 100644 --- a/ush/ioda/bufr2ioda/utils.py +++ b/ush/ioda/bufr2ioda/utils.py @@ -62,7 +62,6 @@ def nc_merge(file_name1, file_name2, target_name): vars1 = grp1[k].variables vars2 = grp2[k].variables for key in vars1: - logger.info(key) var = vars1[key] if vars2.get(key): var2 = vars2[key]